From 5372e72a9884c141874465f0eb8054a4a81da6bb Mon Sep 17 00:00:00 2001 From: Yongye Zhu Date: Mon, 14 Sep 2026 11:57:37 -0700 Subject: [PATCH] [Perf][DSv4.1] Fold the mHC post block into the delayed pre projection (#56633) Signed-off-by: Yongye Zhu --- tests/distributed/test_engram_dp_shard.py | 7 +- tests/kernels/test_mhc_jit_warmup.py | 49 ++- tests/kernels/test_mhc_kernels.py | 248 ++++++++++++++- vllm/model_executor/kernels/mhc/tilelang.py | 285 +++++++++++++++++- .../kernels/mhc/tilelang_kernels.py | 140 ++++++++- vllm/model_executor/kernels/mhc/warmup.py | 22 +- vllm/models/deepseek_v41/nvidia/dspark.py | 2 +- vllm/models/deepseek_v41/nvidia/model.py | 178 +++++++---- 8 files changed, 838 insertions(+), 93 deletions(-) diff --git a/tests/distributed/test_engram_dp_shard.py b/tests/distributed/test_engram_dp_shard.py index 35ffc659d4..c2fa0f857f 100644 --- a/tests/distributed/test_engram_dp_shard.py +++ b/tests/distributed/test_engram_dp_shard.py @@ -491,7 +491,9 @@ def _check_dummy_hash_model_forward( state.forward = lambda *args: ids.unsqueeze(1) class Decoder(SimpleNamespace): - def __call__(self, hidden, positions, input_ids, *args): + # Engram hashes and the mask stay the last positional arguments; + # **kwargs absorbs the decoder's keyword-only flags. + def __call__(self, hidden, positions, input_ids, *args, **kwargs): hashes, keep = args[-2:] if dp_shared_memory and dp_rank == 1: assert hashes is None and keep is None @@ -504,7 +506,8 @@ def _check_dummy_hash_model_forward( if dp_rank == 1: assert torch.all(hashes == engram_ops.DEAD_ID) output = engram.embed(hashes[:, 0]) - return output, None, None, None, None + # Trailing None is previous_aux; this stub captures no aux states. + return output, None, None, None, None, None model = SimpleNamespace( use_mega_moe=False, diff --git a/tests/kernels/test_mhc_jit_warmup.py b/tests/kernels/test_mhc_jit_warmup.py index 28e746c092..512b3e6ba5 100644 --- a/tests/kernels/test_mhc_jit_warmup.py +++ b/tests/kernels/test_mhc_jit_warmup.py @@ -17,6 +17,8 @@ from vllm.model_executor.kernels.mhc.tilelang_kernels import ( MhcFusedTileLangKernel, MhcPostTileLangKernel, MhcPreBigFuseTileLangKernel, + mhc_fused_post_pre_split_config, + require_fused_post_pre_config, ) from vllm.model_executor.warmup import jit_warmup_tilelang_helper @@ -113,17 +115,32 @@ def test_mhc_pre_big_fuse_dispatch_matches_legacy_runtime_config( @pytest.mark.parametrize( - ("num_tokens", "hidden_size", "expected_n_splits", "expected_tile_n"), - [(4, 4096, 8, 2), (4, 8192, 4, 2), (8, 4096, 4, 3)], + ("num_tokens", "hidden_size", "expected_tile_n", "expected_n_splits"), + [ + (1, 4096, 2, 8), + (4, 8192, 2, 8), + (8, 5120, 2, 8), + (16, 5120, 6, 8), + (32, 7168, 6, 8), + ], ) -def test_mhc_fused_dispatch_matches_legacy_runtime_config( +def test_mhc_fused_dispatch_matches_the_launch_config( num_tokens: int, hidden_size: int, - expected_n_splits: int, expected_tile_n: int, + expected_n_splits: int, ) -> None: - kernel = MhcFusedTileLangKernel() + """The compile key must be the config the launch will actually use. + Both read mhc_fused_post_pre_split_config, so this pins the tuned bands + and guards against the two drifting apart again. + """ + kernel = MhcFusedTileLangKernel() + tile_n, n_splits, n_thr = require_fused_post_pre_config( + num_tokens, hidden_size, hc_mult=4 + ) + + assert (tile_n, n_splits) == (expected_tile_n, expected_n_splits) assert kernel.dispatch( num_tokens=num_tokens, hidden_size=hidden_size, @@ -131,11 +148,20 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config( ) == kernel.CompileKey( hidden_size=hidden_size, hc_mult=4, - n_splits=expected_n_splits, - tile_n=expected_tile_n, + n_splits=n_splits, + tile_n=tile_n, + n_thr=n_thr, ) +def test_mhc_fused_declines_shapes_it_cannot_tile() -> None: + """Above the token cutoff, and for a hidden size the block cannot split.""" + assert mhc_fused_post_pre_split_config(33, 5120, 4) is None + assert mhc_fused_post_pre_split_config(1, 5137, 4) is None + with pytest.raises(ValueError, match="does not cover num_tokens"): + require_fused_post_pre_config(33, 5120, 4) + + @pytest.mark.parametrize( ("kernel", "compile_key"), [ @@ -235,8 +261,9 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config( MhcFusedTileLangKernel.CompileKey( hidden_size=4096, hc_mult=4, - n_splits=4, - tile_n=3, + n_splits=8, + tile_n=6, + n_thr=128, ), ), ( @@ -246,6 +273,7 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config( hc_mult=4, n_splits=8, tile_n=2, + n_thr=128, ), ), ( @@ -253,8 +281,9 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config( MhcFusedTileLangKernel.CompileKey( hidden_size=8192, hc_mult=4, - n_splits=4, + n_splits=8, tile_n=2, + n_thr=128, ), ), ( diff --git a/tests/kernels/test_mhc_kernels.py b/tests/kernels/test_mhc_kernels.py index 46f311dfe6..5049815380 100644 --- a/tests/kernels/test_mhc_kernels.py +++ b/tests/kernels/test_mhc_kernels.py @@ -15,8 +15,12 @@ from vllm.model_executor.kernels.mhc.tilelang import ( ) from vllm.model_executor.kernels.mhc.tilelang_kernels import ( _HC_PRENORM_GEMM_TILELANG_KERNEL, + mhc_fused_post_pre_split_config, +) +from vllm.model_executor.kernels.mhc.torch import ( + mhc_post_torch, + mhc_pre_delayed_torch, ) -from vllm.model_executor.kernels.mhc.torch import mhc_pre_delayed_torch from vllm.model_executor.kernels.mhc.triton import hc_collapse_triton from vllm.model_executor.layers.mhc import ( HAS_AITER_MHC, @@ -115,9 +119,9 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch): carried_mixes = [] def make_layer(mix): - def forward(hidden, positions, ids, pre, post, res, residual): + def forward(hidden, positions, ids, pre, post, res, residual, **kwargs): carried_mixes.append(pre) - return hidden, None, None, None, mix + return hidden, None, None, None, mix, None return forward @@ -235,6 +239,171 @@ def test_deepseek_v41_mhc_pre_delayed( torch.testing.assert_close(actual[2], expected[2], atol=1.6e-2, rtol=1e-2) +def mhc_fused_post_pre_delayed_ref( + x: torch.Tensor, + residual: torch.Tensor, + post_layer_mix: torch.Tensor, + comb_res_mix: torch.Tensor, + *mix_args, + pre_mix: torch.Tensor | None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Project the post result before it is rounded to BF16. + + The fused kernel keeps the updated residual streams in registers, so its + projection sees the FP32 post result while the streams it stores are the + BF16 rounding of it. + """ + post = torch.einsum("tij,tih->tjh", comb_res_mix, residual.float()) + post = post + post_layer_mix * x.float().unsqueeze(-2) + return mhc_pre_delayed_torch( + post.bfloat16(), *mix_args, pre_mix=pre_mix, x=post.flatten(1) + ) + + +@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required") +@pytest.mark.parametrize("num_tokens", [0, 1, 8, 128]) +@pytest.mark.parametrize("hidden_size", [4096, 7168]) +@pytest.mark.parametrize("carried", [False, True]) +def test_deepseek_v41_mhc_fused_post_pre_delayed(num_tokens, hidden_size, carried): + """Fold the post block into the delayed pre without changing its outputs.""" + set_random_seed(0) + hc_mult = 4 + mix_size = hc_mult * (hc_mult + 2) + x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE) + residual = torch.randn( + num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=DEVICE + ) + post_layer_mix = torch.rand(num_tokens, hc_mult, 1, device=DEVICE) + comb_res_mix = torch.rand(num_tokens, hc_mult, hc_mult, device=DEVICE) + fn = torch.randn(mix_size, hc_mult * hidden_size, device=DEVICE) * 0.02 + scale = torch.tensor([0.5, 0.25, 1.0], device=DEVICE) + base = torch.randn(mix_size, device=DEVICE) + pre_mix = torch.rand(num_tokens, hc_mult, device=DEVICE) if carried else None + weight = torch.empty(hidden_size, dtype=torch.bfloat16, device=DEVICE) + weight.uniform_(0.5, 1.5) + mix_args = (fn, scale, base, 1e-20, 1e-6, 1e-6, 2.0, 20) + + residual_cur, post, comb, layer_input, next_pre, aux = ( + torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang( + x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, weight, 1e-6 + ) + ) + assert aux.shape == (0, hidden_size) + assert [ + residual_cur.shape, + post.shape, + comb.shape, + layer_input.shape, + next_pre.shape, + ] == [ + (num_tokens, hc_mult, hidden_size), + (num_tokens, hc_mult, 1), + (num_tokens, hc_mult, hc_mult), + (num_tokens, hidden_size), + (num_tokens, hc_mult), + ] + if num_tokens == 0: + return + + # The post mapping and the collapse are what the unfused kernels produce. + residual_ref = torch.ops.vllm.mhc_post_tilelang( + x, residual, post_layer_mix, comb_res_mix + ) + layer_input_ref = mhc_pre_delayed_tilelang( + residual_ref, *mix_args, pre_mix=pre_mix, norm_weight=weight, norm_eps=1e-6 + )[2] + torch.testing.assert_close(residual_cur, residual_ref, atol=0, rtol=0) + torch.testing.assert_close(layer_input, layer_input_ref, atol=0, rtol=0) + + post_ref, comb_ref, _, next_pre_ref = mhc_fused_post_pre_delayed_ref( + x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix=pre_mix + ) + # Above the fused kernel's token count the projection falls back to a + # split-k TF32 GEMM over the stored BF16 streams. + tol = ( + 1e-5 + if mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult) is not None + else 5e-3 + ) + for actual, expected in ( + (post, post_ref), + (comb, comb_ref), + (next_pre, next_pre_ref), + ): + torch.testing.assert_close(actual, expected, atol=tol, rtol=tol) + + # Folding the stream mean into the collapse must not change it, or anything + # else the op returns. + captured = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang( + x, + residual, + post_layer_mix, + comb_res_mix, + *mix_args, + pre_mix, + weight, + 1e-6, + True, + ) + torch.testing.assert_close(captured[5], residual_cur.mean(dim=1), atol=0, rtol=0) + unchanged = (residual_cur, post, comb, layer_input, next_pre) + for with_aux, without in zip(captured[:5], unchanged, strict=True): + torch.testing.assert_close(with_aux, without, atol=0, rtol=0) + + # The unnormalized epilogue has its own fused aux path. + bare = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang( + x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, None, 1e-6, True + ) + torch.testing.assert_close(bare[5], bare[0].mean(dim=1), atol=0, rtol=0) + + # Decode replays this op from a captured graph. + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang( + x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, weight, 1e-6 + ) + graph.replay() + eager = (residual_cur, post, comb, layer_input, next_pre, aux) + for replayed, expected in zip(captured, eager, strict=True): + torch.testing.assert_close(replayed, expected, atol=0, rtol=0) + + +@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required") +@pytest.mark.parametrize("carried", [False, True]) +@pytest.mark.parametrize("capture_aux", [False, True]) +def test_mhc_fused_post_pre_delayed_custom_op_supports_compile(carried, capture_aux): + set_random_seed(0) + x = torch.randn(2, 5120, dtype=torch.bfloat16, device=DEVICE) + residual = torch.randn(2, 4, 5120, dtype=torch.bfloat16, device=DEVICE) + post_layer_mix = torch.rand(2, 4, 1, device=DEVICE) + comb_res_mix = torch.rand(2, 4, 4, device=DEVICE) + fn = torch.randn(24, 20480, device=DEVICE) * 0.02 + pre_mix = torch.rand(2, 4, device=DEVICE) if carried else None + scale = torch.ones(3, device=DEVICE) + base = torch.zeros(24, device=DEVICE) + torch.library.opcheck( + torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang.default, + ( + x, + residual, + post_layer_mix, + comb_res_mix, + fn, + scale, + base, + 1e-20, + 1e-6, + 1e-6, + 2.0, + 20, + pre_mix, + None, + 1e-6, + capture_aux, + ), + ) + + @pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required") @pytest.mark.parametrize("entry", ["broadcast", "pipeline", "residual", "engram"]) def test_deepseek_v41_decoder_mixes_match_torch( @@ -295,15 +464,86 @@ def test_deepseek_v41_decoder_mixes_match_torch( post, res, collapsed, pre = mhc_pre_delayed_torch(*args, **kwargs) return post, res, decoder.attn_norm(collapsed), pre + def fused_reference(x, residual, post_mix, res_mix, *args, capture_aux=False, **kw): + residual = mhc_post_torch(x, residual, post_mix, res_mix) + aux = residual.mean(dim=1) if capture_aux else residual.new_empty(0) + return residual, *reference(residual, *args, **kw), aux + monkeypatch.setattr( "vllm.models.deepseek_v41.nvidia.model.mhc_pre_delayed_tilelang", reference, ) + monkeypatch.setattr( + "vllm.models.deepseek_v41.nvidia.model.mhc_fused_post_pre_delayed_tilelang", + fused_reference, + ) expected = decoder(x, positions, None, **kwargs) - for result, ref in zip(actual, expected, strict=True): + assert actual[-1] is None and expected[-1] is None + for result, ref in zip(actual[:-1], expected[:-1], strict=True): torch.testing.assert_close(result, ref, atol=2e-2, rtol=1e-2) +@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required") +@pytest.mark.parametrize("entry", ["fused", "engram"]) +def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_config): + """Read the aux hidden state back out of the seam that already computed it. + + The draft model's input is the mean over hc streams of the previous + layer's post. That post now runs inside this layer's fused pre, so the + captured value must equal a standalone ``mhc_post_tilelang`` on the same + inputs -- and Engram must not be folded into it, since the injection + happens after the aux consumers read the stream. + """ + set_random_seed(0) + decoder = DeepseekV41DecoderLayer.__new__(DeepseekV41DecoderLayer) + nn.Module.__init__(decoder) + decoder.hc_mult = 4 + decoder.hc_sinkhorn_iters = 20 + decoder.hc_eps = 1e-6 + decoder.rms_norm_eps = 1e-6 + decoder.hc_post_alpha = 2.0 + decoder.use_sequence_parallel = False + decoder.engram = None + from vllm.model_executor.layers.layernorm import RMSNorm + + decoder.attn_norm = decoder.ffn_norm = RMSNorm(5120, 1e-6).to( + device=DEVICE, dtype=torch.bfloat16 + ) + decoder.attn = lambda positions, x, _: x * 0.5 + decoder.ffn = lambda x, input_ids: x * 0.25 + with torch.device(DEVICE): + decoder.hc_attn_fn = torch.randn(24, 20480) * 0.02 + decoder.hc_ffn_fn = torch.randn(24, 20480) * 0.02 + decoder.hc_attn_scale = decoder.hc_ffn_scale = torch.ones(3) + decoder.hc_attn_base = torch.randn(24) + decoder.hc_ffn_base = torch.randn(24) + x = torch.randn(3, 5120, dtype=torch.bfloat16) + kwargs = dict( + pre_mix=torch.rand(3, 4), + residual=torch.randn(3, 4, 5120, dtype=torch.bfloat16), + post_mix=torch.rand(3, 4, 1), + res_mix=torch.rand(3, 4, 4), + ) + if entry == "engram": + + class FakeEngram(nn.Module): + layer_hash_index = 0 + + def forward(self, residual, hashes, mask): + return residual + 0.125 + + decoder.engram = FakeEngram() + kwargs["engram_hashes"] = torch.zeros(3, 1, 1, dtype=torch.int32) + positions = torch.arange(3) + + expected = torch.ops.vllm.mhc_post_tilelang( + x, kwargs["residual"], kwargs["post_mix"], kwargs["res_mix"] + ).mean(dim=1) + previous_aux = decoder(x, positions, None, **kwargs, capture_previous_aux=True)[-1] + torch.testing.assert_close(previous_aux, expected, atol=0, rtol=0) + assert decoder(x, positions, None, **kwargs)[-1] is None + + @pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required") @pytest.mark.parametrize("carried", [False, True]) def test_mhc_pre_delayed_custom_op_supports_compile(carried): diff --git a/vllm/model_executor/kernels/mhc/tilelang.py b/vllm/model_executor/kernels/mhc/tilelang.py index 97ca23b541..cea186e49e 100644 --- a/vllm/model_executor/kernels/mhc/tilelang.py +++ b/vllm/model_executor/kernels/mhc/tilelang.py @@ -194,6 +194,7 @@ def mhc_pre_delayed_tilelang( norm_weight, pre_mix if pre_mix is not None else post, next_pre_mix, + layer_input, hidden_size=hidden_size, rms_eps=rms_eps, hc_pre_eps=hc_pre_eps, @@ -218,6 +219,7 @@ def mhc_pre_delayed_tilelang( layer_input, pre_mix if pre_mix is not None else post, next_pre_mix, + layer_input, hidden_size, rms_eps, hc_pre_eps, @@ -263,6 +265,278 @@ def _mhc_pre_delayed_tilelang_fake( ) +def mhc_fused_post_pre_delayed_tilelang( + x: torch.Tensor, + residual: torch.Tensor, + post_layer_mix: torch.Tensor, + comb_res_mix: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None = None, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 1e-6, + capture_aux: bool = False, +) -> tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor +]: + """Run one mHC post block followed by the next delayed mHC pre block. + + Within the fused kernel's token range the post mapping is folded into the + pre-norm GEMM, so the updated residual streams feed the projection from + registers instead of a second pass over global memory. Above it this runs + the same post kernel and split-k GEMM as the unfused pair. + + Args: + x: BF16 sublayer output of shape (tokens, hidden_size). + residual: BF16 residual streams of shape (tokens, hc_mult, hidden_size). + post_layer_mix: FP32 post coefficients of shape (tokens, hc_mult, 1). + comb_res_mix: FP32 residual coefficients, (tokens, hc_mult, hc_mult). + fn: FP32 projection of shape (hc_mult * (hc_mult + 2), input_size). + hc_scale: FP32 scales of shape (3,). + hc_base: FP32 bias of shape (hc_mult * (hc_mult + 2),). + rms_eps: RMS normalization epsilon. + hc_pre_eps: Pre-mix epsilon. + hc_sinkhorn_eps: Sinkhorn epsilon. + hc_post_mult_value: Post-mix multiplier. + sinkhorn_repeat: Number of Sinkhorn iterations. + pre_mix: FP32 coefficients from the previous sublayer, or None to + select residual stream zero. + norm_weight: Optional BF16 RMSNorm weight for the collapsed input. + norm_eps: RMSNorm epsilon for the collapsed input. + capture_aux: Also return the mean over the post-mapped streams, which + draft models consume as the target's hidden state. It is folded + into the collapse, which already reads those streams. + + Returns: + The post-mapped residual streams, the post and residual coefficients, + the optionally normalized BF16 layer input, the next FP32 pre-mix, and + the BF16 stream mean (empty unless capture_aux), with shapes + (tokens, hc_mult, hidden_size), (tokens, hc_mult, 1), + (tokens, hc_mult, hc_mult), (tokens, hidden_size), (tokens, hc_mult), + and (tokens, hidden_size). + """ + from vllm.model_executor.kernels.mhc.tilelang_kernels import ( + _HC_PRENORM_GEMM_TILELANG_KERNEL, + _MHC_FUSED_TILELANG_KERNEL, + _MHC_POST_TILELANG_KERNEL, + mhc_fused_post_pre_split_config, + mhc_pre_big_fuse_tilelang, + ) + from vllm.model_executor.kernels.mhc.warmup import ( + MHC_PRE_NORM_KERNEL, + compute_mhc_pre_num_splits, + ) + from vllm.utils.deep_gemm import ( + is_deep_gemm_supported, + tf32_hc_prenorm_gemm, + ) + + assert residual.ndim == 3 and residual.dtype == torch.bfloat16 + assert residual.is_contiguous() + num_tokens, hc_mult, hidden_size = residual.shape + input_size = hc_mult * hidden_size + mix_size = hc_mult * (hc_mult + 2) + assert x.shape == (num_tokens, hidden_size) and x.dtype == torch.bfloat16 + assert x.is_contiguous() + assert post_layer_mix.shape[:2] == (num_tokens, hc_mult) + assert post_layer_mix.dtype == torch.float32 and post_layer_mix.is_contiguous() + assert comb_res_mix.shape == (num_tokens, hc_mult, hc_mult) + assert comb_res_mix.dtype == torch.float32 and comb_res_mix.is_contiguous() + assert fn.shape == (mix_size, input_size) and fn.dtype == torch.float32 + assert hc_scale.shape == (3,) and hc_scale.dtype == torch.float32 + assert hc_base.shape == (mix_size,) and hc_base.dtype == torch.float32 + if pre_mix is not None: + assert pre_mix.shape == (num_tokens, hc_mult) + assert pre_mix.dtype == torch.float32 and pre_mix.is_contiguous() + + next_pre_mix = torch.empty( + num_tokens, hc_mult, dtype=torch.float32, device=residual.device + ) + post = torch.empty_like(next_pre_mix) + comb = torch.empty( + num_tokens, hc_mult * hc_mult, dtype=torch.float32, device=residual.device + ) + layer_input = torch.empty( + num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device + ) + aux = torch.empty( + num_tokens if capture_aux else 0, + hidden_size, + dtype=torch.bfloat16, + device=residual.device, + ) + if num_tokens == 0: + return ( + torch.empty_like(residual), + post.unsqueeze(-1), + comb.view(num_tokens, hc_mult, hc_mult), + layer_input, + next_pre_mix, + aux, + ) + + fused_config = mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult) + if fused_config is not None: + mixes, sqrsum, residual_cur = _MHC_FUSED_TILELANG_KERNEL( + comb_res_mix, + residual, + post_layer_mix.view(num_tokens, hc_mult), + x, + fn.view(mix_size, hc_mult, hidden_size), + hc_mult, + hidden_size, + mix_size, + ) + else: + residual_cur = torch.empty_like(residual) + _MHC_POST_TILELANG_KERNEL( + comb_res_mix, + residual, + post_layer_mix.view(num_tokens, hc_mult), + x, + residual_cur, + hc_mult, + hidden_size, + ) + # The delayed epilogue is compiled per bucketed split count, so the + # projection has to use the same bucket rather than the raw estimate. + use_deep_gemm = is_deep_gemm_supported() + n_splits = ( + compute_mhc_pre_num_splits(input_size, num_tokens) if use_deep_gemm else 1 + ) + mixes = torch.empty( + n_splits, num_tokens, mix_size, dtype=torch.float32, device=residual.device + ) + sqrsum = torch.empty( + n_splits, num_tokens, dtype=torch.float32, device=residual.device + ) + residual_cur_2d = residual_cur.view(num_tokens, input_size) + if use_deep_gemm: + tf32_hc_prenorm_gemm(residual_cur_2d, fn, mixes, sqrsum, n_splits) + else: + _HC_PRENORM_GEMM_TILELANG_KERNEL( + residual_cur_2d, + fn, + mixes, + sqrsum, + input_size, + 1, + ) + + outputs = ( + residual_cur, + post.unsqueeze(-1), + comb.view(num_tokens, hc_mult, hc_mult), + layer_input, + next_pre_mix, + aux, + ) + if norm_weight is not None: + assert norm_weight.shape == (hidden_size,) + assert norm_weight.dtype == torch.bfloat16 and norm_weight.is_contiguous() + MHC_PRE_NORM_KERNEL( + mixes, + sqrsum, + hc_scale, + hc_base, + residual_cur, + post, + comb, + layer_input, + norm_weight, + pre_mix if pre_mix is not None else post, + next_pre_mix, + aux if capture_aux else layer_input, + hidden_size=hidden_size, + rms_eps=rms_eps, + hc_pre_eps=hc_pre_eps, + hc_sinkhorn_eps=hc_sinkhorn_eps, + hc_post_mult_value=hc_post_mult_value, + sinkhorn_repeat=sinkhorn_repeat, + norm_eps=norm_eps, + hc_mult=hc_mult, + use_pre_mix_in=pre_mix is not None, + save_pre_mix=True, + rms_numel=input_size, + write_aux=capture_aux, + ) + return outputs + mhc_pre_big_fuse_tilelang( + mixes, + sqrsum, + hc_scale, + hc_base, + residual_cur, + post, + comb, + layer_input, + pre_mix if pre_mix is not None else post, + next_pre_mix, + aux if capture_aux else layer_input, + hidden_size, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + mixes.shape[0], + hc_mult, + use_pre_mix_in=pre_mix is not None, + save_pre_mix=True, + rms_numel=input_size, + write_aux=capture_aux, + ) + return outputs + + +def _mhc_fused_post_pre_delayed_tilelang_fake( + x: torch.Tensor, + residual: torch.Tensor, + post_layer_mix: torch.Tensor, + comb_res_mix: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None = None, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 1e-6, + capture_aux: bool = False, +) -> tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor +]: + num_tokens, hc_mult, hidden_size = residual.shape + return ( + torch.empty_like(residual), + torch.empty( + num_tokens, hc_mult, 1, dtype=torch.float32, device=residual.device + ), + torch.empty( + num_tokens, hc_mult, hc_mult, dtype=torch.float32, device=residual.device + ), + torch.empty( + num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device + ), + torch.empty(num_tokens, hc_mult, dtype=torch.float32, device=residual.device), + torch.empty( + num_tokens if capture_aux else 0, + hidden_size, + dtype=torch.bfloat16, + device=residual.device, + ), + ) + + def mhc_pre_tilelang( residual: torch.Tensor, fn: torch.Tensor, @@ -574,6 +848,7 @@ def mhc_fused_post_pre_tilelang( _MHC_FUSED_TILELANG_KERNEL, _MHC_POST_TILELANG_KERNEL, _MHC_PRE_BIG_FUSE_TILELANG_KERNEL, + mhc_fused_post_pre_split_config, ) assert residual.dtype == torch.bfloat16 @@ -614,7 +889,7 @@ def mhc_fused_post_pre_tilelang( post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult) comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult) - use_small_fma = num_tokens <= 16 + fused_config = mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult) post_mix_cur = torch.empty( num_tokens, @@ -635,7 +910,7 @@ def mhc_fused_post_pre_tilelang( device=residual.device, ) - if use_small_fma: + if fused_config is not None: gemm_out_mul, gemm_out_sqrsum, residual_cur = _MHC_FUSED_TILELANG_KERNEL( comb_res_mix_flat, residual_flat, @@ -815,6 +1090,12 @@ direct_register_custom_op( mutates_args=[], fake_impl=_mhc_pre_delayed_tilelang_fake, ) +direct_register_custom_op( + op_name="mhc_fused_post_pre_delayed_tilelang", + op_func=mhc_fused_post_pre_delayed_tilelang, + mutates_args=[], + fake_impl=_mhc_fused_post_pre_delayed_tilelang_fake, +) direct_register_custom_op( op_name="mhc_pre_tilelang", op_func=mhc_pre_tilelang, diff --git a/vllm/model_executor/kernels/mhc/tilelang_kernels.py b/vllm/model_executor/kernels/mhc/tilelang_kernels.py index b4d3190c0e..9e803d123e 100644 --- a/vllm/model_executor/kernels/mhc/tilelang_kernels.py +++ b/vllm/model_executor/kernels/mhc/tilelang_kernels.py @@ -37,8 +37,67 @@ def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int: return split_k -def _mhc_fused_n_splits(num_tokens: int, hidden_size: int) -> int: - return 8 if num_tokens < 8 and hidden_size <= 4096 else 4 +# ``mhc_fused_tilelang`` splits the hidden size across a fixed thread block, so +# every (tile_n, n_thr) pair is one more kernel to compile. Tuned on GB300 +# (hc_mult 4, hidden_size 5120 and 7168) against the separate post + split-k +# GEMM it replaces; the bands are coarse because the loss against a per-shape +# optimum is ~1-2%, while one fixed config gives up to 18% at the top of the +# range. +_FUSED_POST_PRE_N_THR = 128 +_FUSED_POST_PRE_MAX_TOKENS = 32 + + +def mhc_fused_post_pre_split_config( + num_tokens: int, hidden_size: int, hc_mult: int +) -> tuple[int, int, int] | None: + """Pick ``(tile_n, n_splits, n_thr)`` for the fused post + pre-norm GEMM. + + Returns None when a separate post kernel followed by a split-k GEMM is the + better choice, or when the hidden size does not divide evenly across the + fused kernel's block. One source of truth for the dispatch decision, the + compile key and the launch, which must agree. + """ + if num_tokens > _FUSED_POST_PRE_MAX_TOKENS: + return None + n_thr = _FUSED_POST_PRE_N_THR + if hidden_size % n_thr: + return None + # The projection tiles supply the parallelism a small token count cannot, + # so the tile grows with the batch: 1.3x at one token and 1.03x at 32 + # against a split-k GEMM given the split its own estimator computes. + if num_tokens < 16: + tile_n, n_splits = 2, 8 + else: + tile_n, n_splits = 6, 8 + # The kernel drops projection tiles and k-slices it cannot fill evenly. + mix_size = hc_mult * (hc_mult + 2) + while tile_n > 1 and mix_size % tile_n: + tile_n -= 1 + while n_splits > 1 and hidden_size % (n_splits * n_thr): + n_splits //= 2 + return tile_n, n_splits, n_thr + + +def require_fused_post_pre_config( + num_tokens: int, hidden_size: int, hc_mult: int +) -> tuple[int, int, int]: + """The config for a shape the caller has already committed to fusing.""" + config = mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult) + if config is None: + raise ValueError( + "the fused mHC post + pre-norm GEMM does not cover num_tokens=" + f"{num_tokens} at hidden_size={hidden_size}" + ) + return config + + +def mhc_fused_post_pre_splits(hidden_size: int, hc_mult: int) -> tuple[int, ...]: + """Every split-k factor the fused post + pre-norm GEMM path can pick.""" + configs = ( + mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult) + for num_tokens in range(1, _FUSED_POST_PRE_MAX_TOKENS + 1) + ) + return tuple(sorted({config[1] for config in configs if config is not None})) @tilelang_jit @@ -53,6 +112,7 @@ def mhc_pre_big_fuse_tilelang( layer_input, pre_mix_in, pre_mix_out, + aux_out, hidden_size: int, rms_eps: float, hc_pre_eps: float, @@ -64,6 +124,7 @@ def mhc_pre_big_fuse_tilelang( use_pre_mix_in: bool = False, save_pre_mix: bool = False, rms_numel: int = 0, + write_aux: bool = False, ): """Fuse coefficient generation and residual collapse after the projection. @@ -88,6 +149,7 @@ def mhc_pre_big_fuse_tilelang( pre_mix_in: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type] pre_mix_out: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type] + aux_out: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type] with T.Kernel(num_tokens, threads=96) as i: if ENABLE_PDL: @@ -188,11 +250,24 @@ def mhc_pre_big_fuse_tilelang( ol = T.alloc_fragment(hidden_block, T.float32) T.clear(ol) + if write_aux: + # Aux consumers (draft models) take the plain stream mean + # of the same residual this collapse reads. + aux = T.alloc_fragment(hidden_block, T.float32) + T.clear(aux) for i_hc in T.serial(hc_mult): pre = pre_mix_shared[i_hc] for i1_h in T.Parallel(hidden_block): ol[i1_h] += pre * xl[i_hc, i1_h] + if write_aux: + aux[i1_h] += xl[i_hc, i1_h] + + if write_aux: + for i1_h in T.Parallel(hidden_block): + aux_out[i, i0_h * hidden_block + i1_h] = T.bfloat16( + aux[i1_h] / hc_mult + ) T.copy(ol, layer_input[i, i0_h * hidden_block]) @@ -216,6 +291,7 @@ def mhc_pre_big_fuse_with_norm_tilelang( norm_weight, pre_mix_in, pre_mix_out, + aux_out, hidden_size: int, rms_eps: float, hc_pre_eps: float, @@ -229,6 +305,7 @@ def mhc_pre_big_fuse_with_norm_tilelang( use_pre_mix_in: bool = False, save_pre_mix: bool = False, rms_numel: int = 0, + write_aux: bool = False, ): num_tokens = T.dynamic("num_tokens") hc_mult3 = hc_mult * (2 + hc_mult) @@ -250,6 +327,7 @@ def mhc_pre_big_fuse_with_norm_tilelang( pre_mix_in: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type] pre_mix_out: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type] + aux_out: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type] with T.Kernel(num_tokens, threads=96) as i: rms = T.alloc_fragment(1, T.float32) @@ -345,11 +423,24 @@ def mhc_pre_big_fuse_with_norm_tilelang( ol = T.alloc_fragment(hidden_block, T.float32) T.clear(ol) + if write_aux: + # Aux consumers (draft models) take the plain stream mean + # of the same residual this collapse reads. + aux = T.alloc_fragment(hidden_block, T.float32) + T.clear(aux) for i_hc in T.serial(hc_mult): pre = pre_mix_shared[i_hc] for i1_h in T.Parallel(hidden_block): ol[i1_h] += pre * xl[i_hc, i1_h] + if write_aux: + aux[i1_h] += xl[i_hc, i1_h] + + if write_aux: + for i1_h in T.Parallel(hidden_block): + aux_out[i, i0_h * hidden_block + i1_h] = T.bfloat16( + aux[i1_h] / hc_mult + ) if save_pre_mix: # Keep the BF16 boundary before the delayed input RMSNorm. @@ -1211,9 +1302,17 @@ class MhcPreBigFuseTileLangKernel( middle_args = (residual_out, *output_args, norm_weight) elif compile_key.use_norm_weight: assert norm_weight is not None - middle_args = (*output_args, norm_weight, post_mix, post_mix) + # layer_input stands in for the aux buffer on both epilogues: same + # shape and dtype, and write_aux is off here so it is never written. + middle_args = ( + *output_args, + norm_weight, + post_mix, + post_mix, + layer_input, + ) else: - middle_args = (*output_args, post_mix, post_mix) + middle_args = (*output_args, post_mix, post_mix, layer_input) norm_eps_args = (compile_key.norm_eps,) if compile_key.use_norm_weight else () return ( *common_args, @@ -1247,8 +1346,14 @@ class MhcPreBigFuseTileLangKernel( if use_pre_gemm_splits else n_splits ) - fused_n_splits = _mhc_fused_n_splits(num_tokens, hidden_size) - actual_n_splits = fused_n_splits if use_fused_tilelang else pre_gemm_n_splits + # The epilogue reduces over whatever the fused kernel split the GEMM + # into, so both have to read the same config. + fused_config = mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult) + actual_n_splits = ( + fused_config[1] + if use_fused_tilelang and fused_config is not None + else pre_gemm_n_splits + ) actual_norm_eps = broadcast_norm_eps if is_broadcast else norm_eps actual_use_norm_weight = use_norm_weight or is_broadcast return self.CompileKey( @@ -1540,6 +1645,7 @@ class MhcFusedTileLangKernel( hc_mult: int n_splits: int tile_n: int + n_thr: int @staticmethod def kernel() -> Any: @@ -1552,14 +1658,15 @@ class MhcFusedTileLangKernel( hidden_size: int, hc_mult: int, ) -> CompileKey: - # TODO(gnovack): investigate autotuning these heuristics - tile_n = 2 if num_tokens < 8 else 3 - n_splits = _mhc_fused_n_splits(num_tokens, hidden_size) + # Subscripts rather than an unpack: the warmup tracer parses this body + # and allows only plain assignments before the return. + config = require_fused_post_pre_config(num_tokens, hidden_size, hc_mult) return self.CompileKey( hidden_size=hidden_size, hc_mult=hc_mult, - n_splits=n_splits, - tile_n=tile_n, + n_splits=config[1], + tile_n=config[0], + n_thr=config[2], ) def get_warmup_keys( @@ -1574,11 +1681,11 @@ class MhcFusedTileLangKernel( num_tokens=WarmupIntRange(1, max_tokens + 1), hidden_size=hidden_size, hc_mult=hc_mult, - _when=lambda *, num_tokens: num_tokens <= 16, + _when=lambda *, num_tokens: num_tokens <= _FUSED_POST_PRE_MAX_TOKENS, ) def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: - num_tokens = 1 if compile_key.tile_n == 2 else 8 + num_tokens = 1 if compile_key.tile_n == 2 else 16 hidden_size = compile_key.hidden_size hc_mult = compile_key.hc_mult hc_mult3 = hc_mult * (2 + hc_mult) @@ -1617,8 +1724,9 @@ class MhcFusedTileLangKernel( hc_mult3: int, ) -> TileLangLaunchSpec: num_tokens = residual_in.shape[0] - tile_n = 2 if num_tokens < 8 else 3 - n_splits = _mhc_fused_n_splits(num_tokens, hidden_size) + tile_n, n_splits, n_thr = require_fused_post_pre_config( + num_tokens, hidden_size, hc_mult + ) yp_out = residual_in.new_empty( (n_splits, num_tokens, hc_mult3), dtype=torch.float32 ) @@ -1639,7 +1747,7 @@ class MhcFusedTileLangKernel( hidden_size, hc_mult3, ), - dict(tile_n=tile_n, split_k=n_splits), + dict(n_thr=n_thr, tile_n=tile_n), (yp_out, rp_out, residual_out), ) diff --git a/vllm/model_executor/kernels/mhc/warmup.py b/vllm/model_executor/kernels/mhc/warmup.py index 0abe1a2161..4e6a19781c 100644 --- a/vllm/model_executor/kernels/mhc/warmup.py +++ b/vllm/model_executor/kernels/mhc/warmup.py @@ -36,23 +36,33 @@ class MHCPreNormKernel(VllmJitKernel["MHCPreNormKernel.CompileKey"]): use_pre_mix_in: bool rms_numel: int save_pre_mix: bool = True + write_aux: bool = False kernel: Any = staticmethod(mhc_pre_big_fuse_with_norm_tilelang) def dispatch(self, *, n_splits, **fields) -> CompileKey: # type: ignore[override] return self.CompileKey(n_splits=n_splits, **fields) - def get_warmup_keys(self, *, max_tokens: int, **fields) -> list[CompileKey]: - # The split heuristic changes only at 64-token boundaries. - splits = ( - sorted( + def get_warmup_keys( + self, + *, + max_tokens: int, + extra_splits: Iterable[int] = (), + **fields, + ) -> list[CompileKey]: + # The split heuristic changes only at 64-token boundaries. Callers that + # reach this kernel through another GEMM (the fused post + pre-norm + # GEMM) pass the splits that GEMM picks in extra_splits. + splits = sorted( + set(extra_splits) + | ( { compute_mhc_pre_num_splits(fields["rms_numel"], num_tokens) for num_tokens in range(1, max_tokens + 1, 64) } + if is_deep_gemm_supported() + else {1} ) - if is_deep_gemm_supported() - else [1] ) return self._trace_dispatch(self.dispatch)(n_splits=splits, **fields) diff --git a/vllm/models/deepseek_v41/nvidia/dspark.py b/vllm/models/deepseek_v41/nvidia/dspark.py index 46fe2558ab..3c711681aa 100644 --- a/vllm/models/deepseek_v41/nvidia/dspark.py +++ b/vllm/models/deepseek_v41/nvidia/dspark.py @@ -207,7 +207,7 @@ class DSparkDeepseekV4Model(nn.Module): residual = post_mix = res_mix = pre_mix = None for layer in self.layers: - hidden_states, residual, post_mix, res_mix, pre_mix = layer( + hidden_states, residual, post_mix, res_mix, pre_mix, _ = layer( hidden_states, positions, input_ids, diff --git a/vllm/models/deepseek_v41/nvidia/model.py b/vllm/models/deepseek_v41/nvidia/model.py index df12a9d9bb..f3530544da 100644 --- a/vllm/models/deepseek_v41/nvidia/model.py +++ b/vllm/models/deepseek_v41/nvidia/model.py @@ -20,6 +20,7 @@ from vllm.distributed import ( from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.logger import init_logger from vllm.model_executor.kernels.mhc.tilelang import ( + mhc_fused_post_pre_delayed_tilelang, mhc_post_tilelang, mhc_pre_delayed_tilelang, ) @@ -214,17 +215,45 @@ class DeepseekV4DecoderLayer(nn.Module): self.hc_eps = config.hc_eps self.hc_post_alpha = 2.0 if vllm_config.kernel_config.enable_jit_warmup and current_platform.is_cuda(): + from vllm.model_executor.kernels.mhc.tilelang_kernels import ( + mhc_fused_post_pre_splits, + ) from vllm.model_executor.kernels.mhc.warmup import MHC_PRE_NORM_KERNEL max_tokens = vllm_config.scheduler_config.max_num_batched_tokens if self.use_sequence_parallel: tp_size = vllm_config.parallel_config.tensor_parallel_size max_tokens = (max_tokens + tp_size - 1) // tp_size - for input_size, use_pre_mix in ( - (self.hidden_size, False), - (self.hc_mult * self.hidden_size, False), - (self.hc_mult * self.hidden_size, True), - ): + # The epilogue compiles per projection width and per pre-mix mode. + # The first layer projects the broadcast embedding, so it reads one + # hidden_size-wide row and selects stream zero; every later sublayer + # projects the full hc stream, and collapses it with the pre-mix the + # previous sublayer carried in. Those later shapes also arrive + # through the fused post + pre-norm GEMM, which picks split-k + # factors the token sweep below never produces. + broadcast_embedding = { + "rms_numel": self.hidden_size, + "use_pre_mix_in": False, + "extra_splits": (), + } + hc_stream = { + "rms_numel": self.hc_mult * self.hidden_size, + "extra_splits": mhc_fused_post_pre_splits( + self.hidden_size, self.hc_mult + ), + } + variants = [ + broadcast_embedding, + {**hc_stream, "use_pre_mix_in": False}, + {**hc_stream, "use_pre_mix_in": True}, + ] + if vllm_config.speculative_config is not None: + # Draft setups read aux hidden states out of the same collapse, + # which is its own specialization of the kernel. + variants.append( + {**hc_stream, "use_pre_mix_in": True, "write_aux": True} + ) + for variant in variants: MHC_PRE_NORM_KERNEL.register_warmup( max_tokens=max_tokens, hidden_size=self.hidden_size, @@ -235,8 +264,7 @@ class DeepseekV4DecoderLayer(nn.Module): sinkhorn_repeat=self.hc_sinkhorn_iters, norm_eps=self.rms_norm_eps, hc_mult=self.hc_mult, - use_pre_mix_in=use_pre_mix, - rms_numel=input_size, + **variant, ) mix_hc = (2 + self.hc_mult) * self.hc_mult hc_dim = self.hc_mult * self.hidden_size @@ -295,7 +323,17 @@ class DeepseekV4DecoderLayer(nn.Module): residual: torch.Tensor | None = None, engram_hashes: torch.Tensor | None = None, engram_mask: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + *, + capture_previous_aux: bool = False, + ) -> tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor | None, + ]: + previous_aux: torch.Tensor | None = None # The reference collapses each sublayer's input with the *previous* # sublayer's pre-mix: attention uses the pre-mix carried in (identity # for the first layer), the FFN uses this layer's attention pre-mix. @@ -335,17 +373,19 @@ class DeepseekV4DecoderLayer(nn.Module): norm_weight=self.attn_norm.weight, norm_eps=self.attn_norm.variance_epsilon, ) - else: - residual = mhc_post_tilelang(x, residual, post_mix, res_mix) - if self.engram is not None and engram_hashes is not None: - # Engram injection happens between the previous sublayer's - # post and this block's pre, on the full hc stream, so the - # mix coefficients see the injected stream. - residual = self.engram( - residual, - engram_hashes[:, self.engram.layer_hash_index], - engram_mask, - ) + elif self.engram is not None and engram_hashes is not None: + # Engram injection happens between the previous sublayer's post + # and this block's pre, on the full hc stream, so the mix + # coefficients see the injected stream. The injection also keeps + # the post out of the pre-norm GEMM's fused prologue. + previous_post = mhc_post_tilelang(x, residual, post_mix, res_mix) + if capture_previous_aux: + previous_aux = previous_post.mean(dim=1) + residual = self.engram( + previous_post, + engram_hashes[:, self.engram.layer_hash_index], + engram_mask, + ) post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang( residual, self.hc_attn_fn, @@ -360,6 +400,31 @@ class DeepseekV4DecoderLayer(nn.Module): norm_weight=self.attn_norm.weight, norm_eps=self.attn_norm.variance_epsilon, ) + else: + # The collapse already reads the post-mapped streams, so the mean + # aux consumers want comes out of the same kernel. + residual, post_mix, res_mix, x, attn_pre, aux = ( + mhc_fused_post_pre_delayed_tilelang( + x, + residual, + post_mix, + res_mix, + self.hc_attn_fn, + self.hc_attn_scale, + self.hc_attn_base, + self.rms_norm_eps, + self.hc_eps, + self.hc_eps, + self.hc_post_alpha, + self.hc_sinkhorn_iters, + pre_mix=pre_mix, + norm_weight=self.attn_norm.weight, + norm_eps=self.attn_norm.variance_epsilon, + capture_aux=capture_previous_aux, + ) + ) + if capture_previous_aux: + previous_aux = aux if self.use_sequence_parallel: x = sp_all_gather(x)[: positions.shape[0]] @@ -368,23 +433,27 @@ class DeepseekV4DecoderLayer(nn.Module): if self.use_sequence_parallel: x = sp_reduce_scatter(x) - residual = mhc_post_tilelang(x, residual, post_mix, res_mix) - post_mix, res_mix, x, ffn_pre = mhc_pre_delayed_tilelang( - residual, - self.hc_ffn_fn, - self.hc_ffn_scale, - self.hc_ffn_base, - self.rms_norm_eps, - self.hc_eps, - self.hc_eps, - self.hc_post_alpha, - self.hc_sinkhorn_iters, - pre_mix=attn_pre, - norm_weight=self.ffn_norm.weight, - norm_eps=self.ffn_norm.variance_epsilon, + residual, post_mix, res_mix, x, ffn_pre, _ = ( + mhc_fused_post_pre_delayed_tilelang( + x, + residual, + post_mix, + res_mix, + self.hc_ffn_fn, + self.hc_ffn_scale, + self.hc_ffn_base, + self.rms_norm_eps, + self.hc_eps, + self.hc_eps, + self.hc_post_alpha, + self.hc_sinkhorn_iters, + pre_mix=attn_pre, + norm_weight=self.ffn_norm.weight, + norm_eps=self.ffn_norm.variance_epsilon, + ) ) x = self.ffn(x, input_ids) - return x, residual, post_mix, res_mix, ffn_pre + return x, residual, post_mix, res_mix, ffn_pre, previous_aux class DeepseekV4Model(nn.Module, EagleModelMixin): @@ -631,13 +700,14 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): if not get_pp_group().is_first_rank: assert intermediate_tensors is not None pre_mix = intermediate_tensors["pre_mix"] - aux_hidden_states: list[torch.Tensor] = [] - final_aux_recon: torch.Tensor | None = None # avoid duplicate mhc_post call + # Every layer's post runs inside the next layer's fused pre, so aux + # hidden states are read back from there instead of recomputed. + aux_hidden_by_layer: dict[int, torch.Tensor] = {} for idx, layer in enumerate( islice(self.layers, self.start_layer, self.end_layer), start=self.start_layer, ): - hidden_states, residual, post_mix, res_mix, pre_mix = layer( + hidden_states, residual, post_mix, res_mix, pre_mix, previous_aux = layer( hidden_states, positions, input_ids, @@ -647,25 +717,29 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): residual, engram_hashes, engram_mask, + capture_previous_aux=idx in self.aux_hidden_state_layers, ) - if idx + 1 in self.aux_hidden_state_layers: - # Reconstruct the aux hidden state for draft models - aux_recon = mhc_post_tilelang( - hidden_states, residual, post_mix, res_mix - ) - aux_hidden_state = aux_recon.mean(dim=1) + if previous_aux is not None: + # idx is the one-based id of the layer whose post this is. if self.use_sequence_parallel: - aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens] - aux_hidden_states.append(aux_hidden_state) - final_aux_recon = aux_recon + previous_aux = sp_all_gather(previous_aux)[:full_num_tokens] + aux_hidden_by_layer[idx] = previous_aux if layer is not None: - # Reuse if the last layer was captured as an aux hidden state + # The last layer has no successor to fold its post into. + hidden_states = mhc_post_tilelang( + hidden_states, residual, post_mix, res_mix + ) if self.end_layer in self.aux_hidden_state_layers: - hidden_states = final_aux_recon - else: - hidden_states = mhc_post_tilelang( - hidden_states, residual, post_mix, res_mix - ) + final_aux = hidden_states.mean(dim=1) + if self.use_sequence_parallel: + final_aux = sp_all_gather(final_aux)[:full_num_tokens] + aux_hidden_by_layer[self.end_layer] = final_aux + + aux_hidden_states = [ + aux_hidden_by_layer[layer_id] + for layer_id in self.aux_hidden_state_layers + if layer_id in aux_hidden_by_layer + ] if not get_pp_group().is_last_rank: return IntermediateTensors(