[Perf][DSv4.1] Fold the mHC post block into the delayed pre projection (#56633)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Yongye Zhu
2026-09-14 18:57:37 +00:00
committed by GitHub
parent 56ca9904ce
commit 5372e72a98
8 changed files with 838 additions and 93 deletions
+5 -2
View File
@@ -491,7 +491,9 @@ def _check_dummy_hash_model_forward(
state.forward = lambda *args: ids.unsqueeze(1)
class Decoder(SimpleNamespace):
def __call__(self, hidden, positions, input_ids, *args):
# Engram hashes and the mask stay the last positional arguments;
# **kwargs absorbs the decoder's keyword-only flags.
def __call__(self, hidden, positions, input_ids, *args, **kwargs):
hashes, keep = args[-2:]
if dp_shared_memory and dp_rank == 1:
assert hashes is None and keep is None
@@ -504,7 +506,8 @@ def _check_dummy_hash_model_forward(
if dp_rank == 1:
assert torch.all(hashes == engram_ops.DEAD_ID)
output = engram.embed(hashes[:, 0])
return output, None, None, None, None
# Trailing None is previous_aux; this stub captures no aux states.
return output, None, None, None, None, None
model = SimpleNamespace(
use_mega_moe=False,
+39 -10
View File
@@ -17,6 +17,8 @@ from vllm.model_executor.kernels.mhc.tilelang_kernels import (
MhcFusedTileLangKernel,
MhcPostTileLangKernel,
MhcPreBigFuseTileLangKernel,
mhc_fused_post_pre_split_config,
require_fused_post_pre_config,
)
from vllm.model_executor.warmup import jit_warmup_tilelang_helper
@@ -113,17 +115,32 @@ def test_mhc_pre_big_fuse_dispatch_matches_legacy_runtime_config(
@pytest.mark.parametrize(
("num_tokens", "hidden_size", "expected_n_splits", "expected_tile_n"),
[(4, 4096, 8, 2), (4, 8192, 4, 2), (8, 4096, 4, 3)],
("num_tokens", "hidden_size", "expected_tile_n", "expected_n_splits"),
[
(1, 4096, 2, 8),
(4, 8192, 2, 8),
(8, 5120, 2, 8),
(16, 5120, 6, 8),
(32, 7168, 6, 8),
],
)
def test_mhc_fused_dispatch_matches_legacy_runtime_config(
def test_mhc_fused_dispatch_matches_the_launch_config(
num_tokens: int,
hidden_size: int,
expected_n_splits: int,
expected_tile_n: int,
expected_n_splits: int,
) -> None:
kernel = MhcFusedTileLangKernel()
"""The compile key must be the config the launch will actually use.
Both read mhc_fused_post_pre_split_config, so this pins the tuned bands
and guards against the two drifting apart again.
"""
kernel = MhcFusedTileLangKernel()
tile_n, n_splits, n_thr = require_fused_post_pre_config(
num_tokens, hidden_size, hc_mult=4
)
assert (tile_n, n_splits) == (expected_tile_n, expected_n_splits)
assert kernel.dispatch(
num_tokens=num_tokens,
hidden_size=hidden_size,
@@ -131,11 +148,20 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
) == kernel.CompileKey(
hidden_size=hidden_size,
hc_mult=4,
n_splits=expected_n_splits,
tile_n=expected_tile_n,
n_splits=n_splits,
tile_n=tile_n,
n_thr=n_thr,
)
def test_mhc_fused_declines_shapes_it_cannot_tile() -> None:
"""Above the token cutoff, and for a hidden size the block cannot split."""
assert mhc_fused_post_pre_split_config(33, 5120, 4) is None
assert mhc_fused_post_pre_split_config(1, 5137, 4) is None
with pytest.raises(ValueError, match="does not cover num_tokens"):
require_fused_post_pre_config(33, 5120, 4)
@pytest.mark.parametrize(
("kernel", "compile_key"),
[
@@ -235,8 +261,9 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
MhcFusedTileLangKernel.CompileKey(
hidden_size=4096,
hc_mult=4,
n_splits=4,
tile_n=3,
n_splits=8,
tile_n=6,
n_thr=128,
),
),
(
@@ -246,6 +273,7 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
hc_mult=4,
n_splits=8,
tile_n=2,
n_thr=128,
),
),
(
@@ -253,8 +281,9 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
MhcFusedTileLangKernel.CompileKey(
hidden_size=8192,
hc_mult=4,
n_splits=4,
n_splits=8,
tile_n=2,
n_thr=128,
),
),
(
+244 -4
View File
@@ -15,8 +15,12 @@ from vllm.model_executor.kernels.mhc.tilelang import (
)
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_PRENORM_GEMM_TILELANG_KERNEL,
mhc_fused_post_pre_split_config,
)
from vllm.model_executor.kernels.mhc.torch import (
mhc_post_torch,
mhc_pre_delayed_torch,
)
from vllm.model_executor.kernels.mhc.torch import mhc_pre_delayed_torch
from vllm.model_executor.kernels.mhc.triton import hc_collapse_triton
from vllm.model_executor.layers.mhc import (
HAS_AITER_MHC,
@@ -115,9 +119,9 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
carried_mixes = []
def make_layer(mix):
def forward(hidden, positions, ids, pre, post, res, residual):
def forward(hidden, positions, ids, pre, post, res, residual, **kwargs):
carried_mixes.append(pre)
return hidden, None, None, None, mix
return hidden, None, None, None, mix, None
return forward
@@ -235,6 +239,171 @@ def test_deepseek_v41_mhc_pre_delayed(
torch.testing.assert_close(actual[2], expected[2], atol=1.6e-2, rtol=1e-2)
def mhc_fused_post_pre_delayed_ref(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
*mix_args,
pre_mix: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Project the post result before it is rounded to BF16.
The fused kernel keeps the updated residual streams in registers, so its
projection sees the FP32 post result while the streams it stores are the
BF16 rounding of it.
"""
post = torch.einsum("tij,tih->tjh", comb_res_mix, residual.float())
post = post + post_layer_mix * x.float().unsqueeze(-2)
return mhc_pre_delayed_torch(
post.bfloat16(), *mix_args, pre_mix=pre_mix, x=post.flatten(1)
)
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.parametrize("num_tokens", [0, 1, 8, 128])
@pytest.mark.parametrize("hidden_size", [4096, 7168])
@pytest.mark.parametrize("carried", [False, True])
def test_deepseek_v41_mhc_fused_post_pre_delayed(num_tokens, hidden_size, carried):
"""Fold the post block into the delayed pre without changing its outputs."""
set_random_seed(0)
hc_mult = 4
mix_size = hc_mult * (hc_mult + 2)
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE)
residual = torch.randn(
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=DEVICE
)
post_layer_mix = torch.rand(num_tokens, hc_mult, 1, device=DEVICE)
comb_res_mix = torch.rand(num_tokens, hc_mult, hc_mult, device=DEVICE)
fn = torch.randn(mix_size, hc_mult * hidden_size, device=DEVICE) * 0.02
scale = torch.tensor([0.5, 0.25, 1.0], device=DEVICE)
base = torch.randn(mix_size, device=DEVICE)
pre_mix = torch.rand(num_tokens, hc_mult, device=DEVICE) if carried else None
weight = torch.empty(hidden_size, dtype=torch.bfloat16, device=DEVICE)
weight.uniform_(0.5, 1.5)
mix_args = (fn, scale, base, 1e-20, 1e-6, 1e-6, 2.0, 20)
residual_cur, post, comb, layer_input, next_pre, aux = (
torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, weight, 1e-6
)
)
assert aux.shape == (0, hidden_size)
assert [
residual_cur.shape,
post.shape,
comb.shape,
layer_input.shape,
next_pre.shape,
] == [
(num_tokens, hc_mult, hidden_size),
(num_tokens, hc_mult, 1),
(num_tokens, hc_mult, hc_mult),
(num_tokens, hidden_size),
(num_tokens, hc_mult),
]
if num_tokens == 0:
return
# The post mapping and the collapse are what the unfused kernels produce.
residual_ref = torch.ops.vllm.mhc_post_tilelang(
x, residual, post_layer_mix, comb_res_mix
)
layer_input_ref = mhc_pre_delayed_tilelang(
residual_ref, *mix_args, pre_mix=pre_mix, norm_weight=weight, norm_eps=1e-6
)[2]
torch.testing.assert_close(residual_cur, residual_ref, atol=0, rtol=0)
torch.testing.assert_close(layer_input, layer_input_ref, atol=0, rtol=0)
post_ref, comb_ref, _, next_pre_ref = mhc_fused_post_pre_delayed_ref(
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix=pre_mix
)
# Above the fused kernel's token count the projection falls back to a
# split-k TF32 GEMM over the stored BF16 streams.
tol = (
1e-5
if mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult) is not None
else 5e-3
)
for actual, expected in (
(post, post_ref),
(comb, comb_ref),
(next_pre, next_pre_ref),
):
torch.testing.assert_close(actual, expected, atol=tol, rtol=tol)
# Folding the stream mean into the collapse must not change it, or anything
# else the op returns.
captured = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
x,
residual,
post_layer_mix,
comb_res_mix,
*mix_args,
pre_mix,
weight,
1e-6,
True,
)
torch.testing.assert_close(captured[5], residual_cur.mean(dim=1), atol=0, rtol=0)
unchanged = (residual_cur, post, comb, layer_input, next_pre)
for with_aux, without in zip(captured[:5], unchanged, strict=True):
torch.testing.assert_close(with_aux, without, atol=0, rtol=0)
# The unnormalized epilogue has its own fused aux path.
bare = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, None, 1e-6, True
)
torch.testing.assert_close(bare[5], bare[0].mean(dim=1), atol=0, rtol=0)
# Decode replays this op from a captured graph.
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, weight, 1e-6
)
graph.replay()
eager = (residual_cur, post, comb, layer_input, next_pre, aux)
for replayed, expected in zip(captured, eager, strict=True):
torch.testing.assert_close(replayed, expected, atol=0, rtol=0)
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.parametrize("carried", [False, True])
@pytest.mark.parametrize("capture_aux", [False, True])
def test_mhc_fused_post_pre_delayed_custom_op_supports_compile(carried, capture_aux):
set_random_seed(0)
x = torch.randn(2, 5120, dtype=torch.bfloat16, device=DEVICE)
residual = torch.randn(2, 4, 5120, dtype=torch.bfloat16, device=DEVICE)
post_layer_mix = torch.rand(2, 4, 1, device=DEVICE)
comb_res_mix = torch.rand(2, 4, 4, device=DEVICE)
fn = torch.randn(24, 20480, device=DEVICE) * 0.02
pre_mix = torch.rand(2, 4, device=DEVICE) if carried else None
scale = torch.ones(3, device=DEVICE)
base = torch.zeros(24, device=DEVICE)
torch.library.opcheck(
torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang.default,
(
x,
residual,
post_layer_mix,
comb_res_mix,
fn,
scale,
base,
1e-20,
1e-6,
1e-6,
2.0,
20,
pre_mix,
None,
1e-6,
capture_aux,
),
)
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.parametrize("entry", ["broadcast", "pipeline", "residual", "engram"])
def test_deepseek_v41_decoder_mixes_match_torch(
@@ -295,15 +464,86 @@ def test_deepseek_v41_decoder_mixes_match_torch(
post, res, collapsed, pre = mhc_pre_delayed_torch(*args, **kwargs)
return post, res, decoder.attn_norm(collapsed), pre
def fused_reference(x, residual, post_mix, res_mix, *args, capture_aux=False, **kw):
residual = mhc_post_torch(x, residual, post_mix, res_mix)
aux = residual.mean(dim=1) if capture_aux else residual.new_empty(0)
return residual, *reference(residual, *args, **kw), aux
monkeypatch.setattr(
"vllm.models.deepseek_v41.nvidia.model.mhc_pre_delayed_tilelang",
reference,
)
monkeypatch.setattr(
"vllm.models.deepseek_v41.nvidia.model.mhc_fused_post_pre_delayed_tilelang",
fused_reference,
)
expected = decoder(x, positions, None, **kwargs)
for result, ref in zip(actual, expected, strict=True):
assert actual[-1] is None and expected[-1] is None
for result, ref in zip(actual[:-1], expected[:-1], strict=True):
torch.testing.assert_close(result, ref, atol=2e-2, rtol=1e-2)
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.parametrize("entry", ["fused", "engram"])
def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_config):
"""Read the aux hidden state back out of the seam that already computed it.
The draft model's input is the mean over hc streams of the previous
layer's post. That post now runs inside this layer's fused pre, so the
captured value must equal a standalone ``mhc_post_tilelang`` on the same
inputs -- and Engram must not be folded into it, since the injection
happens after the aux consumers read the stream.
"""
set_random_seed(0)
decoder = DeepseekV41DecoderLayer.__new__(DeepseekV41DecoderLayer)
nn.Module.__init__(decoder)
decoder.hc_mult = 4
decoder.hc_sinkhorn_iters = 20
decoder.hc_eps = 1e-6
decoder.rms_norm_eps = 1e-6
decoder.hc_post_alpha = 2.0
decoder.use_sequence_parallel = False
decoder.engram = None
from vllm.model_executor.layers.layernorm import RMSNorm
decoder.attn_norm = decoder.ffn_norm = RMSNorm(5120, 1e-6).to(
device=DEVICE, dtype=torch.bfloat16
)
decoder.attn = lambda positions, x, _: x * 0.5
decoder.ffn = lambda x, input_ids: x * 0.25
with torch.device(DEVICE):
decoder.hc_attn_fn = torch.randn(24, 20480) * 0.02
decoder.hc_ffn_fn = torch.randn(24, 20480) * 0.02
decoder.hc_attn_scale = decoder.hc_ffn_scale = torch.ones(3)
decoder.hc_attn_base = torch.randn(24)
decoder.hc_ffn_base = torch.randn(24)
x = torch.randn(3, 5120, dtype=torch.bfloat16)
kwargs = dict(
pre_mix=torch.rand(3, 4),
residual=torch.randn(3, 4, 5120, dtype=torch.bfloat16),
post_mix=torch.rand(3, 4, 1),
res_mix=torch.rand(3, 4, 4),
)
if entry == "engram":
class FakeEngram(nn.Module):
layer_hash_index = 0
def forward(self, residual, hashes, mask):
return residual + 0.125
decoder.engram = FakeEngram()
kwargs["engram_hashes"] = torch.zeros(3, 1, 1, dtype=torch.int32)
positions = torch.arange(3)
expected = torch.ops.vllm.mhc_post_tilelang(
x, kwargs["residual"], kwargs["post_mix"], kwargs["res_mix"]
).mean(dim=1)
previous_aux = decoder(x, positions, None, **kwargs, capture_previous_aux=True)[-1]
torch.testing.assert_close(previous_aux, expected, atol=0, rtol=0)
assert decoder(x, positions, None, **kwargs)[-1] is None
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.parametrize("carried", [False, True])
def test_mhc_pre_delayed_custom_op_supports_compile(carried):
+283 -2
View File
@@ -194,6 +194,7 @@ def mhc_pre_delayed_tilelang(
norm_weight,
pre_mix if pre_mix is not None else post,
next_pre_mix,
layer_input,
hidden_size=hidden_size,
rms_eps=rms_eps,
hc_pre_eps=hc_pre_eps,
@@ -218,6 +219,7 @@ def mhc_pre_delayed_tilelang(
layer_input,
pre_mix if pre_mix is not None else post,
next_pre_mix,
layer_input,
hidden_size,
rms_eps,
hc_pre_eps,
@@ -263,6 +265,278 @@ def _mhc_pre_delayed_tilelang_fake(
)
def mhc_fused_post_pre_delayed_tilelang(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
capture_aux: bool = False,
) -> tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
]:
"""Run one mHC post block followed by the next delayed mHC pre block.
Within the fused kernel's token range the post mapping is folded into the
pre-norm GEMM, so the updated residual streams feed the projection from
registers instead of a second pass over global memory. Above it this runs
the same post kernel and split-k GEMM as the unfused pair.
Args:
x: BF16 sublayer output of shape (tokens, hidden_size).
residual: BF16 residual streams of shape (tokens, hc_mult, hidden_size).
post_layer_mix: FP32 post coefficients of shape (tokens, hc_mult, 1).
comb_res_mix: FP32 residual coefficients, (tokens, hc_mult, hc_mult).
fn: FP32 projection of shape (hc_mult * (hc_mult + 2), input_size).
hc_scale: FP32 scales of shape (3,).
hc_base: FP32 bias of shape (hc_mult * (hc_mult + 2),).
rms_eps: RMS normalization epsilon.
hc_pre_eps: Pre-mix epsilon.
hc_sinkhorn_eps: Sinkhorn epsilon.
hc_post_mult_value: Post-mix multiplier.
sinkhorn_repeat: Number of Sinkhorn iterations.
pre_mix: FP32 coefficients from the previous sublayer, or None to
select residual stream zero.
norm_weight: Optional BF16 RMSNorm weight for the collapsed input.
norm_eps: RMSNorm epsilon for the collapsed input.
capture_aux: Also return the mean over the post-mapped streams, which
draft models consume as the target's hidden state. It is folded
into the collapse, which already reads those streams.
Returns:
The post-mapped residual streams, the post and residual coefficients,
the optionally normalized BF16 layer input, the next FP32 pre-mix, and
the BF16 stream mean (empty unless capture_aux), with shapes
(tokens, hc_mult, hidden_size), (tokens, hc_mult, 1),
(tokens, hc_mult, hc_mult), (tokens, hidden_size), (tokens, hc_mult),
and (tokens, hidden_size).
"""
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
_HC_PRENORM_GEMM_TILELANG_KERNEL,
_MHC_FUSED_TILELANG_KERNEL,
_MHC_POST_TILELANG_KERNEL,
mhc_fused_post_pre_split_config,
mhc_pre_big_fuse_tilelang,
)
from vllm.model_executor.kernels.mhc.warmup import (
MHC_PRE_NORM_KERNEL,
compute_mhc_pre_num_splits,
)
from vllm.utils.deep_gemm import (
is_deep_gemm_supported,
tf32_hc_prenorm_gemm,
)
assert residual.ndim == 3 and residual.dtype == torch.bfloat16
assert residual.is_contiguous()
num_tokens, hc_mult, hidden_size = residual.shape
input_size = hc_mult * hidden_size
mix_size = hc_mult * (hc_mult + 2)
assert x.shape == (num_tokens, hidden_size) and x.dtype == torch.bfloat16
assert x.is_contiguous()
assert post_layer_mix.shape[:2] == (num_tokens, hc_mult)
assert post_layer_mix.dtype == torch.float32 and post_layer_mix.is_contiguous()
assert comb_res_mix.shape == (num_tokens, hc_mult, hc_mult)
assert comb_res_mix.dtype == torch.float32 and comb_res_mix.is_contiguous()
assert fn.shape == (mix_size, input_size) and fn.dtype == torch.float32
assert hc_scale.shape == (3,) and hc_scale.dtype == torch.float32
assert hc_base.shape == (mix_size,) and hc_base.dtype == torch.float32
if pre_mix is not None:
assert pre_mix.shape == (num_tokens, hc_mult)
assert pre_mix.dtype == torch.float32 and pre_mix.is_contiguous()
next_pre_mix = torch.empty(
num_tokens, hc_mult, dtype=torch.float32, device=residual.device
)
post = torch.empty_like(next_pre_mix)
comb = torch.empty(
num_tokens, hc_mult * hc_mult, dtype=torch.float32, device=residual.device
)
layer_input = torch.empty(
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
)
aux = torch.empty(
num_tokens if capture_aux else 0,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
)
if num_tokens == 0:
return (
torch.empty_like(residual),
post.unsqueeze(-1),
comb.view(num_tokens, hc_mult, hc_mult),
layer_input,
next_pre_mix,
aux,
)
fused_config = mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult)
if fused_config is not None:
mixes, sqrsum, residual_cur = _MHC_FUSED_TILELANG_KERNEL(
comb_res_mix,
residual,
post_layer_mix.view(num_tokens, hc_mult),
x,
fn.view(mix_size, hc_mult, hidden_size),
hc_mult,
hidden_size,
mix_size,
)
else:
residual_cur = torch.empty_like(residual)
_MHC_POST_TILELANG_KERNEL(
comb_res_mix,
residual,
post_layer_mix.view(num_tokens, hc_mult),
x,
residual_cur,
hc_mult,
hidden_size,
)
# The delayed epilogue is compiled per bucketed split count, so the
# projection has to use the same bucket rather than the raw estimate.
use_deep_gemm = is_deep_gemm_supported()
n_splits = (
compute_mhc_pre_num_splits(input_size, num_tokens) if use_deep_gemm else 1
)
mixes = torch.empty(
n_splits, num_tokens, mix_size, dtype=torch.float32, device=residual.device
)
sqrsum = torch.empty(
n_splits, num_tokens, dtype=torch.float32, device=residual.device
)
residual_cur_2d = residual_cur.view(num_tokens, input_size)
if use_deep_gemm:
tf32_hc_prenorm_gemm(residual_cur_2d, fn, mixes, sqrsum, n_splits)
else:
_HC_PRENORM_GEMM_TILELANG_KERNEL(
residual_cur_2d,
fn,
mixes,
sqrsum,
input_size,
1,
)
outputs = (
residual_cur,
post.unsqueeze(-1),
comb.view(num_tokens, hc_mult, hc_mult),
layer_input,
next_pre_mix,
aux,
)
if norm_weight is not None:
assert norm_weight.shape == (hidden_size,)
assert norm_weight.dtype == torch.bfloat16 and norm_weight.is_contiguous()
MHC_PRE_NORM_KERNEL(
mixes,
sqrsum,
hc_scale,
hc_base,
residual_cur,
post,
comb,
layer_input,
norm_weight,
pre_mix if pre_mix is not None else post,
next_pre_mix,
aux if capture_aux else layer_input,
hidden_size=hidden_size,
rms_eps=rms_eps,
hc_pre_eps=hc_pre_eps,
hc_sinkhorn_eps=hc_sinkhorn_eps,
hc_post_mult_value=hc_post_mult_value,
sinkhorn_repeat=sinkhorn_repeat,
norm_eps=norm_eps,
hc_mult=hc_mult,
use_pre_mix_in=pre_mix is not None,
save_pre_mix=True,
rms_numel=input_size,
write_aux=capture_aux,
)
return outputs
mhc_pre_big_fuse_tilelang(
mixes,
sqrsum,
hc_scale,
hc_base,
residual_cur,
post,
comb,
layer_input,
pre_mix if pre_mix is not None else post,
next_pre_mix,
aux if capture_aux else layer_input,
hidden_size,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
mixes.shape[0],
hc_mult,
use_pre_mix_in=pre_mix is not None,
save_pre_mix=True,
rms_numel=input_size,
write_aux=capture_aux,
)
return outputs
def _mhc_fused_post_pre_delayed_tilelang_fake(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
capture_aux: bool = False,
) -> tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
]:
num_tokens, hc_mult, hidden_size = residual.shape
return (
torch.empty_like(residual),
torch.empty(
num_tokens, hc_mult, 1, dtype=torch.float32, device=residual.device
),
torch.empty(
num_tokens, hc_mult, hc_mult, dtype=torch.float32, device=residual.device
),
torch.empty(
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
),
torch.empty(num_tokens, hc_mult, dtype=torch.float32, device=residual.device),
torch.empty(
num_tokens if capture_aux else 0,
hidden_size,
dtype=torch.bfloat16,
device=residual.device,
),
)
def mhc_pre_tilelang(
residual: torch.Tensor,
fn: torch.Tensor,
@@ -574,6 +848,7 @@ def mhc_fused_post_pre_tilelang(
_MHC_FUSED_TILELANG_KERNEL,
_MHC_POST_TILELANG_KERNEL,
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
mhc_fused_post_pre_split_config,
)
assert residual.dtype == torch.bfloat16
@@ -614,7 +889,7 @@ def mhc_fused_post_pre_tilelang(
post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult)
comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult)
use_small_fma = num_tokens <= 16
fused_config = mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult)
post_mix_cur = torch.empty(
num_tokens,
@@ -635,7 +910,7 @@ def mhc_fused_post_pre_tilelang(
device=residual.device,
)
if use_small_fma:
if fused_config is not None:
gemm_out_mul, gemm_out_sqrsum, residual_cur = _MHC_FUSED_TILELANG_KERNEL(
comb_res_mix_flat,
residual_flat,
@@ -815,6 +1090,12 @@ direct_register_custom_op(
mutates_args=[],
fake_impl=_mhc_pre_delayed_tilelang_fake,
)
direct_register_custom_op(
op_name="mhc_fused_post_pre_delayed_tilelang",
op_func=mhc_fused_post_pre_delayed_tilelang,
mutates_args=[],
fake_impl=_mhc_fused_post_pre_delayed_tilelang_fake,
)
direct_register_custom_op(
op_name="mhc_pre_tilelang",
op_func=mhc_pre_tilelang,
@@ -37,8 +37,67 @@ def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int:
return split_k
def _mhc_fused_n_splits(num_tokens: int, hidden_size: int) -> int:
return 8 if num_tokens < 8 and hidden_size <= 4096 else 4
# ``mhc_fused_tilelang`` splits the hidden size across a fixed thread block, so
# every (tile_n, n_thr) pair is one more kernel to compile. Tuned on GB300
# (hc_mult 4, hidden_size 5120 and 7168) against the separate post + split-k
# GEMM it replaces; the bands are coarse because the loss against a per-shape
# optimum is ~1-2%, while one fixed config gives up to 18% at the top of the
# range.
_FUSED_POST_PRE_N_THR = 128
_FUSED_POST_PRE_MAX_TOKENS = 32
def mhc_fused_post_pre_split_config(
num_tokens: int, hidden_size: int, hc_mult: int
) -> tuple[int, int, int] | None:
"""Pick ``(tile_n, n_splits, n_thr)`` for the fused post + pre-norm GEMM.
Returns None when a separate post kernel followed by a split-k GEMM is the
better choice, or when the hidden size does not divide evenly across the
fused kernel's block. One source of truth for the dispatch decision, the
compile key and the launch, which must agree.
"""
if num_tokens > _FUSED_POST_PRE_MAX_TOKENS:
return None
n_thr = _FUSED_POST_PRE_N_THR
if hidden_size % n_thr:
return None
# The projection tiles supply the parallelism a small token count cannot,
# so the tile grows with the batch: 1.3x at one token and 1.03x at 32
# against a split-k GEMM given the split its own estimator computes.
if num_tokens < 16:
tile_n, n_splits = 2, 8
else:
tile_n, n_splits = 6, 8
# The kernel drops projection tiles and k-slices it cannot fill evenly.
mix_size = hc_mult * (hc_mult + 2)
while tile_n > 1 and mix_size % tile_n:
tile_n -= 1
while n_splits > 1 and hidden_size % (n_splits * n_thr):
n_splits //= 2
return tile_n, n_splits, n_thr
def require_fused_post_pre_config(
num_tokens: int, hidden_size: int, hc_mult: int
) -> tuple[int, int, int]:
"""The config for a shape the caller has already committed to fusing."""
config = mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult)
if config is None:
raise ValueError(
"the fused mHC post + pre-norm GEMM does not cover num_tokens="
f"{num_tokens} at hidden_size={hidden_size}"
)
return config
def mhc_fused_post_pre_splits(hidden_size: int, hc_mult: int) -> tuple[int, ...]:
"""Every split-k factor the fused post + pre-norm GEMM path can pick."""
configs = (
mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult)
for num_tokens in range(1, _FUSED_POST_PRE_MAX_TOKENS + 1)
)
return tuple(sorted({config[1] for config in configs if config is not None}))
@tilelang_jit
@@ -53,6 +112,7 @@ def mhc_pre_big_fuse_tilelang(
layer_input,
pre_mix_in,
pre_mix_out,
aux_out,
hidden_size: int,
rms_eps: float,
hc_pre_eps: float,
@@ -64,6 +124,7 @@ def mhc_pre_big_fuse_tilelang(
use_pre_mix_in: bool = False,
save_pre_mix: bool = False,
rms_numel: int = 0,
write_aux: bool = False,
):
"""Fuse coefficient generation and residual collapse after the projection.
@@ -88,6 +149,7 @@ def mhc_pre_big_fuse_tilelang(
pre_mix_in: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type]
pre_mix_out: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type]
aux_out: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
with T.Kernel(num_tokens, threads=96) as i:
if ENABLE_PDL:
@@ -188,11 +250,24 @@ def mhc_pre_big_fuse_tilelang(
ol = T.alloc_fragment(hidden_block, T.float32)
T.clear(ol)
if write_aux:
# Aux consumers (draft models) take the plain stream mean
# of the same residual this collapse reads.
aux = T.alloc_fragment(hidden_block, T.float32)
T.clear(aux)
for i_hc in T.serial(hc_mult):
pre = pre_mix_shared[i_hc]
for i1_h in T.Parallel(hidden_block):
ol[i1_h] += pre * xl[i_hc, i1_h]
if write_aux:
aux[i1_h] += xl[i_hc, i1_h]
if write_aux:
for i1_h in T.Parallel(hidden_block):
aux_out[i, i0_h * hidden_block + i1_h] = T.bfloat16(
aux[i1_h] / hc_mult
)
T.copy(ol, layer_input[i, i0_h * hidden_block])
@@ -216,6 +291,7 @@ def mhc_pre_big_fuse_with_norm_tilelang(
norm_weight,
pre_mix_in,
pre_mix_out,
aux_out,
hidden_size: int,
rms_eps: float,
hc_pre_eps: float,
@@ -229,6 +305,7 @@ def mhc_pre_big_fuse_with_norm_tilelang(
use_pre_mix_in: bool = False,
save_pre_mix: bool = False,
rms_numel: int = 0,
write_aux: bool = False,
):
num_tokens = T.dynamic("num_tokens")
hc_mult3 = hc_mult * (2 + hc_mult)
@@ -250,6 +327,7 @@ def mhc_pre_big_fuse_with_norm_tilelang(
pre_mix_in: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type]
pre_mix_out: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type]
aux_out: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
with T.Kernel(num_tokens, threads=96) as i:
rms = T.alloc_fragment(1, T.float32)
@@ -345,11 +423,24 @@ def mhc_pre_big_fuse_with_norm_tilelang(
ol = T.alloc_fragment(hidden_block, T.float32)
T.clear(ol)
if write_aux:
# Aux consumers (draft models) take the plain stream mean
# of the same residual this collapse reads.
aux = T.alloc_fragment(hidden_block, T.float32)
T.clear(aux)
for i_hc in T.serial(hc_mult):
pre = pre_mix_shared[i_hc]
for i1_h in T.Parallel(hidden_block):
ol[i1_h] += pre * xl[i_hc, i1_h]
if write_aux:
aux[i1_h] += xl[i_hc, i1_h]
if write_aux:
for i1_h in T.Parallel(hidden_block):
aux_out[i, i0_h * hidden_block + i1_h] = T.bfloat16(
aux[i1_h] / hc_mult
)
if save_pre_mix:
# Keep the BF16 boundary before the delayed input RMSNorm.
@@ -1211,9 +1302,17 @@ class MhcPreBigFuseTileLangKernel(
middle_args = (residual_out, *output_args, norm_weight)
elif compile_key.use_norm_weight:
assert norm_weight is not None
middle_args = (*output_args, norm_weight, post_mix, post_mix)
# layer_input stands in for the aux buffer on both epilogues: same
# shape and dtype, and write_aux is off here so it is never written.
middle_args = (
*output_args,
norm_weight,
post_mix,
post_mix,
layer_input,
)
else:
middle_args = (*output_args, post_mix, post_mix)
middle_args = (*output_args, post_mix, post_mix, layer_input)
norm_eps_args = (compile_key.norm_eps,) if compile_key.use_norm_weight else ()
return (
*common_args,
@@ -1247,8 +1346,14 @@ class MhcPreBigFuseTileLangKernel(
if use_pre_gemm_splits
else n_splits
)
fused_n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
actual_n_splits = fused_n_splits if use_fused_tilelang else pre_gemm_n_splits
# The epilogue reduces over whatever the fused kernel split the GEMM
# into, so both have to read the same config.
fused_config = mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult)
actual_n_splits = (
fused_config[1]
if use_fused_tilelang and fused_config is not None
else pre_gemm_n_splits
)
actual_norm_eps = broadcast_norm_eps if is_broadcast else norm_eps
actual_use_norm_weight = use_norm_weight or is_broadcast
return self.CompileKey(
@@ -1540,6 +1645,7 @@ class MhcFusedTileLangKernel(
hc_mult: int
n_splits: int
tile_n: int
n_thr: int
@staticmethod
def kernel() -> Any:
@@ -1552,14 +1658,15 @@ class MhcFusedTileLangKernel(
hidden_size: int,
hc_mult: int,
) -> CompileKey:
# TODO(gnovack): investigate autotuning these heuristics
tile_n = 2 if num_tokens < 8 else 3
n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
# Subscripts rather than an unpack: the warmup tracer parses this body
# and allows only plain assignments before the return.
config = require_fused_post_pre_config(num_tokens, hidden_size, hc_mult)
return self.CompileKey(
hidden_size=hidden_size,
hc_mult=hc_mult,
n_splits=n_splits,
tile_n=tile_n,
n_splits=config[1],
tile_n=config[0],
n_thr=config[2],
)
def get_warmup_keys(
@@ -1574,11 +1681,11 @@ class MhcFusedTileLangKernel(
num_tokens=WarmupIntRange(1, max_tokens + 1),
hidden_size=hidden_size,
hc_mult=hc_mult,
_when=lambda *, num_tokens: num_tokens <= 16,
_when=lambda *, num_tokens: num_tokens <= _FUSED_POST_PRE_MAX_TOKENS,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
num_tokens = 1 if compile_key.tile_n == 2 else 8
num_tokens = 1 if compile_key.tile_n == 2 else 16
hidden_size = compile_key.hidden_size
hc_mult = compile_key.hc_mult
hc_mult3 = hc_mult * (2 + hc_mult)
@@ -1617,8 +1724,9 @@ class MhcFusedTileLangKernel(
hc_mult3: int,
) -> TileLangLaunchSpec:
num_tokens = residual_in.shape[0]
tile_n = 2 if num_tokens < 8 else 3
n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
tile_n, n_splits, n_thr = require_fused_post_pre_config(
num_tokens, hidden_size, hc_mult
)
yp_out = residual_in.new_empty(
(n_splits, num_tokens, hc_mult3), dtype=torch.float32
)
@@ -1639,7 +1747,7 @@ class MhcFusedTileLangKernel(
hidden_size,
hc_mult3,
),
dict(tile_n=tile_n, split_k=n_splits),
dict(n_thr=n_thr, tile_n=tile_n),
(yp_out, rp_out, residual_out),
)
+16 -6
View File
@@ -36,23 +36,33 @@ class MHCPreNormKernel(VllmJitKernel["MHCPreNormKernel.CompileKey"]):
use_pre_mix_in: bool
rms_numel: int
save_pre_mix: bool = True
write_aux: bool = False
kernel: Any = staticmethod(mhc_pre_big_fuse_with_norm_tilelang)
def dispatch(self, *, n_splits, **fields) -> CompileKey: # type: ignore[override]
return self.CompileKey(n_splits=n_splits, **fields)
def get_warmup_keys(self, *, max_tokens: int, **fields) -> list[CompileKey]:
# The split heuristic changes only at 64-token boundaries.
splits = (
sorted(
def get_warmup_keys(
self,
*,
max_tokens: int,
extra_splits: Iterable[int] = (),
**fields,
) -> list[CompileKey]:
# The split heuristic changes only at 64-token boundaries. Callers that
# reach this kernel through another GEMM (the fused post + pre-norm
# GEMM) pass the splits that GEMM picks in extra_splits.
splits = sorted(
set(extra_splits)
| (
{
compute_mhc_pre_num_splits(fields["rms_numel"], num_tokens)
for num_tokens in range(1, max_tokens + 1, 64)
}
if is_deep_gemm_supported()
else {1}
)
if is_deep_gemm_supported()
else [1]
)
return self._trace_dispatch(self.dispatch)(n_splits=splits, **fields)
+1 -1
View File
@@ -207,7 +207,7 @@ class DSparkDeepseekV4Model(nn.Module):
residual = post_mix = res_mix = pre_mix = None
for layer in self.layers:
hidden_states, residual, post_mix, res_mix, pre_mix = layer(
hidden_states, residual, post_mix, res_mix, pre_mix, _ = layer(
hidden_states,
positions,
input_ids,
+126 -52
View File
@@ -20,6 +20,7 @@ from vllm.distributed import (
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
mhc_fused_post_pre_delayed_tilelang,
mhc_post_tilelang,
mhc_pre_delayed_tilelang,
)
@@ -214,17 +215,45 @@ class DeepseekV4DecoderLayer(nn.Module):
self.hc_eps = config.hc_eps
self.hc_post_alpha = 2.0
if vllm_config.kernel_config.enable_jit_warmup and current_platform.is_cuda():
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
mhc_fused_post_pre_splits,
)
from vllm.model_executor.kernels.mhc.warmup import MHC_PRE_NORM_KERNEL
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
if self.use_sequence_parallel:
tp_size = vllm_config.parallel_config.tensor_parallel_size
max_tokens = (max_tokens + tp_size - 1) // tp_size
for input_size, use_pre_mix in (
(self.hidden_size, False),
(self.hc_mult * self.hidden_size, False),
(self.hc_mult * self.hidden_size, True),
):
# The epilogue compiles per projection width and per pre-mix mode.
# The first layer projects the broadcast embedding, so it reads one
# hidden_size-wide row and selects stream zero; every later sublayer
# projects the full hc stream, and collapses it with the pre-mix the
# previous sublayer carried in. Those later shapes also arrive
# through the fused post + pre-norm GEMM, which picks split-k
# factors the token sweep below never produces.
broadcast_embedding = {
"rms_numel": self.hidden_size,
"use_pre_mix_in": False,
"extra_splits": (),
}
hc_stream = {
"rms_numel": self.hc_mult * self.hidden_size,
"extra_splits": mhc_fused_post_pre_splits(
self.hidden_size, self.hc_mult
),
}
variants = [
broadcast_embedding,
{**hc_stream, "use_pre_mix_in": False},
{**hc_stream, "use_pre_mix_in": True},
]
if vllm_config.speculative_config is not None:
# Draft setups read aux hidden states out of the same collapse,
# which is its own specialization of the kernel.
variants.append(
{**hc_stream, "use_pre_mix_in": True, "write_aux": True}
)
for variant in variants:
MHC_PRE_NORM_KERNEL.register_warmup(
max_tokens=max_tokens,
hidden_size=self.hidden_size,
@@ -235,8 +264,7 @@ class DeepseekV4DecoderLayer(nn.Module):
sinkhorn_repeat=self.hc_sinkhorn_iters,
norm_eps=self.rms_norm_eps,
hc_mult=self.hc_mult,
use_pre_mix_in=use_pre_mix,
rms_numel=input_size,
**variant,
)
mix_hc = (2 + self.hc_mult) * self.hc_mult
hc_dim = self.hc_mult * self.hidden_size
@@ -295,7 +323,17 @@ class DeepseekV4DecoderLayer(nn.Module):
residual: torch.Tensor | None = None,
engram_hashes: torch.Tensor | None = None,
engram_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
*,
capture_previous_aux: bool = False,
) -> tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor | None,
]:
previous_aux: torch.Tensor | None = None
# The reference collapses each sublayer's input with the *previous*
# sublayer's pre-mix: attention uses the pre-mix carried in (identity
# for the first layer), the FFN uses this layer's attention pre-mix.
@@ -335,17 +373,19 @@ class DeepseekV4DecoderLayer(nn.Module):
norm_weight=self.attn_norm.weight,
norm_eps=self.attn_norm.variance_epsilon,
)
else:
residual = mhc_post_tilelang(x, residual, post_mix, res_mix)
if self.engram is not None and engram_hashes is not None:
# Engram injection happens between the previous sublayer's
# post and this block's pre, on the full hc stream, so the
# mix coefficients see the injected stream.
residual = self.engram(
residual,
engram_hashes[:, self.engram.layer_hash_index],
engram_mask,
)
elif self.engram is not None and engram_hashes is not None:
# Engram injection happens between the previous sublayer's post
# and this block's pre, on the full hc stream, so the mix
# coefficients see the injected stream. The injection also keeps
# the post out of the pre-norm GEMM's fused prologue.
previous_post = mhc_post_tilelang(x, residual, post_mix, res_mix)
if capture_previous_aux:
previous_aux = previous_post.mean(dim=1)
residual = self.engram(
previous_post,
engram_hashes[:, self.engram.layer_hash_index],
engram_mask,
)
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang(
residual,
self.hc_attn_fn,
@@ -360,6 +400,31 @@ class DeepseekV4DecoderLayer(nn.Module):
norm_weight=self.attn_norm.weight,
norm_eps=self.attn_norm.variance_epsilon,
)
else:
# The collapse already reads the post-mapped streams, so the mean
# aux consumers want comes out of the same kernel.
residual, post_mix, res_mix, x, attn_pre, aux = (
mhc_fused_post_pre_delayed_tilelang(
x,
residual,
post_mix,
res_mix,
self.hc_attn_fn,
self.hc_attn_scale,
self.hc_attn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
pre_mix=pre_mix,
norm_weight=self.attn_norm.weight,
norm_eps=self.attn_norm.variance_epsilon,
capture_aux=capture_previous_aux,
)
)
if capture_previous_aux:
previous_aux = aux
if self.use_sequence_parallel:
x = sp_all_gather(x)[: positions.shape[0]]
@@ -368,23 +433,27 @@ class DeepseekV4DecoderLayer(nn.Module):
if self.use_sequence_parallel:
x = sp_reduce_scatter(x)
residual = mhc_post_tilelang(x, residual, post_mix, res_mix)
post_mix, res_mix, x, ffn_pre = mhc_pre_delayed_tilelang(
residual,
self.hc_ffn_fn,
self.hc_ffn_scale,
self.hc_ffn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
pre_mix=attn_pre,
norm_weight=self.ffn_norm.weight,
norm_eps=self.ffn_norm.variance_epsilon,
residual, post_mix, res_mix, x, ffn_pre, _ = (
mhc_fused_post_pre_delayed_tilelang(
x,
residual,
post_mix,
res_mix,
self.hc_ffn_fn,
self.hc_ffn_scale,
self.hc_ffn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
pre_mix=attn_pre,
norm_weight=self.ffn_norm.weight,
norm_eps=self.ffn_norm.variance_epsilon,
)
)
x = self.ffn(x, input_ids)
return x, residual, post_mix, res_mix, ffn_pre
return x, residual, post_mix, res_mix, ffn_pre, previous_aux
class DeepseekV4Model(nn.Module, EagleModelMixin):
@@ -631,13 +700,14 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
if not get_pp_group().is_first_rank:
assert intermediate_tensors is not None
pre_mix = intermediate_tensors["pre_mix"]
aux_hidden_states: list[torch.Tensor] = []
final_aux_recon: torch.Tensor | None = None # avoid duplicate mhc_post call
# Every layer's post runs inside the next layer's fused pre, so aux
# hidden states are read back from there instead of recomputed.
aux_hidden_by_layer: dict[int, torch.Tensor] = {}
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
):
hidden_states, residual, post_mix, res_mix, pre_mix = layer(
hidden_states, residual, post_mix, res_mix, pre_mix, previous_aux = layer(
hidden_states,
positions,
input_ids,
@@ -647,25 +717,29 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
residual,
engram_hashes,
engram_mask,
capture_previous_aux=idx in self.aux_hidden_state_layers,
)
if idx + 1 in self.aux_hidden_state_layers:
# Reconstruct the aux hidden state for draft models
aux_recon = mhc_post_tilelang(
hidden_states, residual, post_mix, res_mix
)
aux_hidden_state = aux_recon.mean(dim=1)
if previous_aux is not None:
# idx is the one-based id of the layer whose post this is.
if self.use_sequence_parallel:
aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens]
aux_hidden_states.append(aux_hidden_state)
final_aux_recon = aux_recon
previous_aux = sp_all_gather(previous_aux)[:full_num_tokens]
aux_hidden_by_layer[idx] = previous_aux
if layer is not None:
# Reuse if the last layer was captured as an aux hidden state
# The last layer has no successor to fold its post into.
hidden_states = mhc_post_tilelang(
hidden_states, residual, post_mix, res_mix
)
if self.end_layer in self.aux_hidden_state_layers:
hidden_states = final_aux_recon
else:
hidden_states = mhc_post_tilelang(
hidden_states, residual, post_mix, res_mix
)
final_aux = hidden_states.mean(dim=1)
if self.use_sequence_parallel:
final_aux = sp_all_gather(final_aux)[:full_num_tokens]
aux_hidden_by_layer[self.end_layer] = final_aux
aux_hidden_states = [
aux_hidden_by_layer[layer_id]
for layer_id in self.aux_hidden_state_layers
if layer_id in aux_hidden_by_layer
]
if not get_pp_group().is_last_rank:
return IntermediateTensors(