[Perf][DSv4.1] Fold the mHC post block into the delayed pre projection (#56633)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -491,7 +491,9 @@ def _check_dummy_hash_model_forward(
|
||||
state.forward = lambda *args: ids.unsqueeze(1)
|
||||
|
||||
class Decoder(SimpleNamespace):
|
||||
def __call__(self, hidden, positions, input_ids, *args):
|
||||
# Engram hashes and the mask stay the last positional arguments;
|
||||
# **kwargs absorbs the decoder's keyword-only flags.
|
||||
def __call__(self, hidden, positions, input_ids, *args, **kwargs):
|
||||
hashes, keep = args[-2:]
|
||||
if dp_shared_memory and dp_rank == 1:
|
||||
assert hashes is None and keep is None
|
||||
@@ -504,7 +506,8 @@ def _check_dummy_hash_model_forward(
|
||||
if dp_rank == 1:
|
||||
assert torch.all(hashes == engram_ops.DEAD_ID)
|
||||
output = engram.embed(hashes[:, 0])
|
||||
return output, None, None, None, None
|
||||
# Trailing None is previous_aux; this stub captures no aux states.
|
||||
return output, None, None, None, None, None
|
||||
|
||||
model = SimpleNamespace(
|
||||
use_mega_moe=False,
|
||||
|
||||
@@ -17,6 +17,8 @@ from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
MhcFusedTileLangKernel,
|
||||
MhcPostTileLangKernel,
|
||||
MhcPreBigFuseTileLangKernel,
|
||||
mhc_fused_post_pre_split_config,
|
||||
require_fused_post_pre_config,
|
||||
)
|
||||
from vllm.model_executor.warmup import jit_warmup_tilelang_helper
|
||||
|
||||
@@ -113,17 +115,32 @@ def test_mhc_pre_big_fuse_dispatch_matches_legacy_runtime_config(
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "hidden_size", "expected_n_splits", "expected_tile_n"),
|
||||
[(4, 4096, 8, 2), (4, 8192, 4, 2), (8, 4096, 4, 3)],
|
||||
("num_tokens", "hidden_size", "expected_tile_n", "expected_n_splits"),
|
||||
[
|
||||
(1, 4096, 2, 8),
|
||||
(4, 8192, 2, 8),
|
||||
(8, 5120, 2, 8),
|
||||
(16, 5120, 6, 8),
|
||||
(32, 7168, 6, 8),
|
||||
],
|
||||
)
|
||||
def test_mhc_fused_dispatch_matches_legacy_runtime_config(
|
||||
def test_mhc_fused_dispatch_matches_the_launch_config(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
expected_n_splits: int,
|
||||
expected_tile_n: int,
|
||||
expected_n_splits: int,
|
||||
) -> None:
|
||||
kernel = MhcFusedTileLangKernel()
|
||||
"""The compile key must be the config the launch will actually use.
|
||||
|
||||
Both read mhc_fused_post_pre_split_config, so this pins the tuned bands
|
||||
and guards against the two drifting apart again.
|
||||
"""
|
||||
kernel = MhcFusedTileLangKernel()
|
||||
tile_n, n_splits, n_thr = require_fused_post_pre_config(
|
||||
num_tokens, hidden_size, hc_mult=4
|
||||
)
|
||||
|
||||
assert (tile_n, n_splits) == (expected_tile_n, expected_n_splits)
|
||||
assert kernel.dispatch(
|
||||
num_tokens=num_tokens,
|
||||
hidden_size=hidden_size,
|
||||
@@ -131,11 +148,20 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
|
||||
) == kernel.CompileKey(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=4,
|
||||
n_splits=expected_n_splits,
|
||||
tile_n=expected_tile_n,
|
||||
n_splits=n_splits,
|
||||
tile_n=tile_n,
|
||||
n_thr=n_thr,
|
||||
)
|
||||
|
||||
|
||||
def test_mhc_fused_declines_shapes_it_cannot_tile() -> None:
|
||||
"""Above the token cutoff, and for a hidden size the block cannot split."""
|
||||
assert mhc_fused_post_pre_split_config(33, 5120, 4) is None
|
||||
assert mhc_fused_post_pre_split_config(1, 5137, 4) is None
|
||||
with pytest.raises(ValueError, match="does not cover num_tokens"):
|
||||
require_fused_post_pre_config(33, 5120, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kernel", "compile_key"),
|
||||
[
|
||||
@@ -235,8 +261,9 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
|
||||
MhcFusedTileLangKernel.CompileKey(
|
||||
hidden_size=4096,
|
||||
hc_mult=4,
|
||||
n_splits=4,
|
||||
tile_n=3,
|
||||
n_splits=8,
|
||||
tile_n=6,
|
||||
n_thr=128,
|
||||
),
|
||||
),
|
||||
(
|
||||
@@ -246,6 +273,7 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
|
||||
hc_mult=4,
|
||||
n_splits=8,
|
||||
tile_n=2,
|
||||
n_thr=128,
|
||||
),
|
||||
),
|
||||
(
|
||||
@@ -253,8 +281,9 @@ def test_mhc_fused_dispatch_matches_legacy_runtime_config(
|
||||
MhcFusedTileLangKernel.CompileKey(
|
||||
hidden_size=8192,
|
||||
hc_mult=4,
|
||||
n_splits=4,
|
||||
n_splits=8,
|
||||
tile_n=2,
|
||||
n_thr=128,
|
||||
),
|
||||
),
|
||||
(
|
||||
|
||||
@@ -15,8 +15,12 @@ from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
)
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL,
|
||||
mhc_fused_post_pre_split_config,
|
||||
)
|
||||
from vllm.model_executor.kernels.mhc.torch import (
|
||||
mhc_post_torch,
|
||||
mhc_pre_delayed_torch,
|
||||
)
|
||||
from vllm.model_executor.kernels.mhc.torch import mhc_pre_delayed_torch
|
||||
from vllm.model_executor.kernels.mhc.triton import hc_collapse_triton
|
||||
from vllm.model_executor.layers.mhc import (
|
||||
HAS_AITER_MHC,
|
||||
@@ -115,9 +119,9 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
|
||||
carried_mixes = []
|
||||
|
||||
def make_layer(mix):
|
||||
def forward(hidden, positions, ids, pre, post, res, residual):
|
||||
def forward(hidden, positions, ids, pre, post, res, residual, **kwargs):
|
||||
carried_mixes.append(pre)
|
||||
return hidden, None, None, None, mix
|
||||
return hidden, None, None, None, mix, None
|
||||
|
||||
return forward
|
||||
|
||||
@@ -235,6 +239,171 @@ def test_deepseek_v41_mhc_pre_delayed(
|
||||
torch.testing.assert_close(actual[2], expected[2], atol=1.6e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def mhc_fused_post_pre_delayed_ref(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
*mix_args,
|
||||
pre_mix: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Project the post result before it is rounded to BF16.
|
||||
|
||||
The fused kernel keeps the updated residual streams in registers, so its
|
||||
projection sees the FP32 post result while the streams it stores are the
|
||||
BF16 rounding of it.
|
||||
"""
|
||||
post = torch.einsum("tij,tih->tjh", comb_res_mix, residual.float())
|
||||
post = post + post_layer_mix * x.float().unsqueeze(-2)
|
||||
return mhc_pre_delayed_torch(
|
||||
post.bfloat16(), *mix_args, pre_mix=pre_mix, x=post.flatten(1)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize("num_tokens", [0, 1, 8, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [4096, 7168])
|
||||
@pytest.mark.parametrize("carried", [False, True])
|
||||
def test_deepseek_v41_mhc_fused_post_pre_delayed(num_tokens, hidden_size, carried):
|
||||
"""Fold the post block into the delayed pre without changing its outputs."""
|
||||
set_random_seed(0)
|
||||
hc_mult = 4
|
||||
mix_size = hc_mult * (hc_mult + 2)
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE)
|
||||
residual = torch.randn(
|
||||
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
post_layer_mix = torch.rand(num_tokens, hc_mult, 1, device=DEVICE)
|
||||
comb_res_mix = torch.rand(num_tokens, hc_mult, hc_mult, device=DEVICE)
|
||||
fn = torch.randn(mix_size, hc_mult * hidden_size, device=DEVICE) * 0.02
|
||||
scale = torch.tensor([0.5, 0.25, 1.0], device=DEVICE)
|
||||
base = torch.randn(mix_size, device=DEVICE)
|
||||
pre_mix = torch.rand(num_tokens, hc_mult, device=DEVICE) if carried else None
|
||||
weight = torch.empty(hidden_size, dtype=torch.bfloat16, device=DEVICE)
|
||||
weight.uniform_(0.5, 1.5)
|
||||
mix_args = (fn, scale, base, 1e-20, 1e-6, 1e-6, 2.0, 20)
|
||||
|
||||
residual_cur, post, comb, layer_input, next_pre, aux = (
|
||||
torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
|
||||
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, weight, 1e-6
|
||||
)
|
||||
)
|
||||
assert aux.shape == (0, hidden_size)
|
||||
assert [
|
||||
residual_cur.shape,
|
||||
post.shape,
|
||||
comb.shape,
|
||||
layer_input.shape,
|
||||
next_pre.shape,
|
||||
] == [
|
||||
(num_tokens, hc_mult, hidden_size),
|
||||
(num_tokens, hc_mult, 1),
|
||||
(num_tokens, hc_mult, hc_mult),
|
||||
(num_tokens, hidden_size),
|
||||
(num_tokens, hc_mult),
|
||||
]
|
||||
if num_tokens == 0:
|
||||
return
|
||||
|
||||
# The post mapping and the collapse are what the unfused kernels produce.
|
||||
residual_ref = torch.ops.vllm.mhc_post_tilelang(
|
||||
x, residual, post_layer_mix, comb_res_mix
|
||||
)
|
||||
layer_input_ref = mhc_pre_delayed_tilelang(
|
||||
residual_ref, *mix_args, pre_mix=pre_mix, norm_weight=weight, norm_eps=1e-6
|
||||
)[2]
|
||||
torch.testing.assert_close(residual_cur, residual_ref, atol=0, rtol=0)
|
||||
torch.testing.assert_close(layer_input, layer_input_ref, atol=0, rtol=0)
|
||||
|
||||
post_ref, comb_ref, _, next_pre_ref = mhc_fused_post_pre_delayed_ref(
|
||||
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix=pre_mix
|
||||
)
|
||||
# Above the fused kernel's token count the projection falls back to a
|
||||
# split-k TF32 GEMM over the stored BF16 streams.
|
||||
tol = (
|
||||
1e-5
|
||||
if mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult) is not None
|
||||
else 5e-3
|
||||
)
|
||||
for actual, expected in (
|
||||
(post, post_ref),
|
||||
(comb, comb_ref),
|
||||
(next_pre, next_pre_ref),
|
||||
):
|
||||
torch.testing.assert_close(actual, expected, atol=tol, rtol=tol)
|
||||
|
||||
# Folding the stream mean into the collapse must not change it, or anything
|
||||
# else the op returns.
|
||||
captured = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
*mix_args,
|
||||
pre_mix,
|
||||
weight,
|
||||
1e-6,
|
||||
True,
|
||||
)
|
||||
torch.testing.assert_close(captured[5], residual_cur.mean(dim=1), atol=0, rtol=0)
|
||||
unchanged = (residual_cur, post, comb, layer_input, next_pre)
|
||||
for with_aux, without in zip(captured[:5], unchanged, strict=True):
|
||||
torch.testing.assert_close(with_aux, without, atol=0, rtol=0)
|
||||
|
||||
# The unnormalized epilogue has its own fused aux path.
|
||||
bare = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
|
||||
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, None, 1e-6, True
|
||||
)
|
||||
torch.testing.assert_close(bare[5], bare[0].mean(dim=1), atol=0, rtol=0)
|
||||
|
||||
# Decode replays this op from a captured graph.
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang(
|
||||
x, residual, post_layer_mix, comb_res_mix, *mix_args, pre_mix, weight, 1e-6
|
||||
)
|
||||
graph.replay()
|
||||
eager = (residual_cur, post, comb, layer_input, next_pre, aux)
|
||||
for replayed, expected in zip(captured, eager, strict=True):
|
||||
torch.testing.assert_close(replayed, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize("carried", [False, True])
|
||||
@pytest.mark.parametrize("capture_aux", [False, True])
|
||||
def test_mhc_fused_post_pre_delayed_custom_op_supports_compile(carried, capture_aux):
|
||||
set_random_seed(0)
|
||||
x = torch.randn(2, 5120, dtype=torch.bfloat16, device=DEVICE)
|
||||
residual = torch.randn(2, 4, 5120, dtype=torch.bfloat16, device=DEVICE)
|
||||
post_layer_mix = torch.rand(2, 4, 1, device=DEVICE)
|
||||
comb_res_mix = torch.rand(2, 4, 4, device=DEVICE)
|
||||
fn = torch.randn(24, 20480, device=DEVICE) * 0.02
|
||||
pre_mix = torch.rand(2, 4, device=DEVICE) if carried else None
|
||||
scale = torch.ones(3, device=DEVICE)
|
||||
base = torch.zeros(24, device=DEVICE)
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm.mhc_fused_post_pre_delayed_tilelang.default,
|
||||
(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
fn,
|
||||
scale,
|
||||
base,
|
||||
1e-20,
|
||||
1e-6,
|
||||
1e-6,
|
||||
2.0,
|
||||
20,
|
||||
pre_mix,
|
||||
None,
|
||||
1e-6,
|
||||
capture_aux,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize("entry", ["broadcast", "pipeline", "residual", "engram"])
|
||||
def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
@@ -295,15 +464,86 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
post, res, collapsed, pre = mhc_pre_delayed_torch(*args, **kwargs)
|
||||
return post, res, decoder.attn_norm(collapsed), pre
|
||||
|
||||
def fused_reference(x, residual, post_mix, res_mix, *args, capture_aux=False, **kw):
|
||||
residual = mhc_post_torch(x, residual, post_mix, res_mix)
|
||||
aux = residual.mean(dim=1) if capture_aux else residual.new_empty(0)
|
||||
return residual, *reference(residual, *args, **kw), aux
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v41.nvidia.model.mhc_pre_delayed_tilelang",
|
||||
reference,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v41.nvidia.model.mhc_fused_post_pre_delayed_tilelang",
|
||||
fused_reference,
|
||||
)
|
||||
expected = decoder(x, positions, None, **kwargs)
|
||||
for result, ref in zip(actual, expected, strict=True):
|
||||
assert actual[-1] is None and expected[-1] is None
|
||||
for result, ref in zip(actual[:-1], expected[:-1], strict=True):
|
||||
torch.testing.assert_close(result, ref, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize("entry", ["fused", "engram"])
|
||||
def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_config):
|
||||
"""Read the aux hidden state back out of the seam that already computed it.
|
||||
|
||||
The draft model's input is the mean over hc streams of the previous
|
||||
layer's post. That post now runs inside this layer's fused pre, so the
|
||||
captured value must equal a standalone ``mhc_post_tilelang`` on the same
|
||||
inputs -- and Engram must not be folded into it, since the injection
|
||||
happens after the aux consumers read the stream.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
decoder = DeepseekV41DecoderLayer.__new__(DeepseekV41DecoderLayer)
|
||||
nn.Module.__init__(decoder)
|
||||
decoder.hc_mult = 4
|
||||
decoder.hc_sinkhorn_iters = 20
|
||||
decoder.hc_eps = 1e-6
|
||||
decoder.rms_norm_eps = 1e-6
|
||||
decoder.hc_post_alpha = 2.0
|
||||
decoder.use_sequence_parallel = False
|
||||
decoder.engram = None
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
|
||||
decoder.attn_norm = decoder.ffn_norm = RMSNorm(5120, 1e-6).to(
|
||||
device=DEVICE, dtype=torch.bfloat16
|
||||
)
|
||||
decoder.attn = lambda positions, x, _: x * 0.5
|
||||
decoder.ffn = lambda x, input_ids: x * 0.25
|
||||
with torch.device(DEVICE):
|
||||
decoder.hc_attn_fn = torch.randn(24, 20480) * 0.02
|
||||
decoder.hc_ffn_fn = torch.randn(24, 20480) * 0.02
|
||||
decoder.hc_attn_scale = decoder.hc_ffn_scale = torch.ones(3)
|
||||
decoder.hc_attn_base = torch.randn(24)
|
||||
decoder.hc_ffn_base = torch.randn(24)
|
||||
x = torch.randn(3, 5120, dtype=torch.bfloat16)
|
||||
kwargs = dict(
|
||||
pre_mix=torch.rand(3, 4),
|
||||
residual=torch.randn(3, 4, 5120, dtype=torch.bfloat16),
|
||||
post_mix=torch.rand(3, 4, 1),
|
||||
res_mix=torch.rand(3, 4, 4),
|
||||
)
|
||||
if entry == "engram":
|
||||
|
||||
class FakeEngram(nn.Module):
|
||||
layer_hash_index = 0
|
||||
|
||||
def forward(self, residual, hashes, mask):
|
||||
return residual + 0.125
|
||||
|
||||
decoder.engram = FakeEngram()
|
||||
kwargs["engram_hashes"] = torch.zeros(3, 1, 1, dtype=torch.int32)
|
||||
positions = torch.arange(3)
|
||||
|
||||
expected = torch.ops.vllm.mhc_post_tilelang(
|
||||
x, kwargs["residual"], kwargs["post_mix"], kwargs["res_mix"]
|
||||
).mean(dim=1)
|
||||
previous_aux = decoder(x, positions, None, **kwargs, capture_previous_aux=True)[-1]
|
||||
torch.testing.assert_close(previous_aux, expected, atol=0, rtol=0)
|
||||
assert decoder(x, positions, None, **kwargs)[-1] is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize("carried", [False, True])
|
||||
def test_mhc_pre_delayed_custom_op_supports_compile(carried):
|
||||
|
||||
@@ -194,6 +194,7 @@ def mhc_pre_delayed_tilelang(
|
||||
norm_weight,
|
||||
pre_mix if pre_mix is not None else post,
|
||||
next_pre_mix,
|
||||
layer_input,
|
||||
hidden_size=hidden_size,
|
||||
rms_eps=rms_eps,
|
||||
hc_pre_eps=hc_pre_eps,
|
||||
@@ -218,6 +219,7 @@ def mhc_pre_delayed_tilelang(
|
||||
layer_input,
|
||||
pre_mix if pre_mix is not None else post,
|
||||
next_pre_mix,
|
||||
layer_input,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
@@ -263,6 +265,278 @@ def _mhc_pre_delayed_tilelang_fake(
|
||||
)
|
||||
|
||||
|
||||
def mhc_fused_post_pre_delayed_tilelang(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
capture_aux: bool = False,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
|
||||
]:
|
||||
"""Run one mHC post block followed by the next delayed mHC pre block.
|
||||
|
||||
Within the fused kernel's token range the post mapping is folded into the
|
||||
pre-norm GEMM, so the updated residual streams feed the projection from
|
||||
registers instead of a second pass over global memory. Above it this runs
|
||||
the same post kernel and split-k GEMM as the unfused pair.
|
||||
|
||||
Args:
|
||||
x: BF16 sublayer output of shape (tokens, hidden_size).
|
||||
residual: BF16 residual streams of shape (tokens, hc_mult, hidden_size).
|
||||
post_layer_mix: FP32 post coefficients of shape (tokens, hc_mult, 1).
|
||||
comb_res_mix: FP32 residual coefficients, (tokens, hc_mult, hc_mult).
|
||||
fn: FP32 projection of shape (hc_mult * (hc_mult + 2), input_size).
|
||||
hc_scale: FP32 scales of shape (3,).
|
||||
hc_base: FP32 bias of shape (hc_mult * (hc_mult + 2),).
|
||||
rms_eps: RMS normalization epsilon.
|
||||
hc_pre_eps: Pre-mix epsilon.
|
||||
hc_sinkhorn_eps: Sinkhorn epsilon.
|
||||
hc_post_mult_value: Post-mix multiplier.
|
||||
sinkhorn_repeat: Number of Sinkhorn iterations.
|
||||
pre_mix: FP32 coefficients from the previous sublayer, or None to
|
||||
select residual stream zero.
|
||||
norm_weight: Optional BF16 RMSNorm weight for the collapsed input.
|
||||
norm_eps: RMSNorm epsilon for the collapsed input.
|
||||
capture_aux: Also return the mean over the post-mapped streams, which
|
||||
draft models consume as the target's hidden state. It is folded
|
||||
into the collapse, which already reads those streams.
|
||||
|
||||
Returns:
|
||||
The post-mapped residual streams, the post and residual coefficients,
|
||||
the optionally normalized BF16 layer input, the next FP32 pre-mix, and
|
||||
the BF16 stream mean (empty unless capture_aux), with shapes
|
||||
(tokens, hc_mult, hidden_size), (tokens, hc_mult, 1),
|
||||
(tokens, hc_mult, hc_mult), (tokens, hidden_size), (tokens, hc_mult),
|
||||
and (tokens, hidden_size).
|
||||
"""
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL,
|
||||
_MHC_FUSED_TILELANG_KERNEL,
|
||||
_MHC_POST_TILELANG_KERNEL,
|
||||
mhc_fused_post_pre_split_config,
|
||||
mhc_pre_big_fuse_tilelang,
|
||||
)
|
||||
from vllm.model_executor.kernels.mhc.warmup import (
|
||||
MHC_PRE_NORM_KERNEL,
|
||||
compute_mhc_pre_num_splits,
|
||||
)
|
||||
from vllm.utils.deep_gemm import (
|
||||
is_deep_gemm_supported,
|
||||
tf32_hc_prenorm_gemm,
|
||||
)
|
||||
|
||||
assert residual.ndim == 3 and residual.dtype == torch.bfloat16
|
||||
assert residual.is_contiguous()
|
||||
num_tokens, hc_mult, hidden_size = residual.shape
|
||||
input_size = hc_mult * hidden_size
|
||||
mix_size = hc_mult * (hc_mult + 2)
|
||||
assert x.shape == (num_tokens, hidden_size) and x.dtype == torch.bfloat16
|
||||
assert x.is_contiguous()
|
||||
assert post_layer_mix.shape[:2] == (num_tokens, hc_mult)
|
||||
assert post_layer_mix.dtype == torch.float32 and post_layer_mix.is_contiguous()
|
||||
assert comb_res_mix.shape == (num_tokens, hc_mult, hc_mult)
|
||||
assert comb_res_mix.dtype == torch.float32 and comb_res_mix.is_contiguous()
|
||||
assert fn.shape == (mix_size, input_size) and fn.dtype == torch.float32
|
||||
assert hc_scale.shape == (3,) and hc_scale.dtype == torch.float32
|
||||
assert hc_base.shape == (mix_size,) and hc_base.dtype == torch.float32
|
||||
if pre_mix is not None:
|
||||
assert pre_mix.shape == (num_tokens, hc_mult)
|
||||
assert pre_mix.dtype == torch.float32 and pre_mix.is_contiguous()
|
||||
|
||||
next_pre_mix = torch.empty(
|
||||
num_tokens, hc_mult, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
post = torch.empty_like(next_pre_mix)
|
||||
comb = torch.empty(
|
||||
num_tokens, hc_mult * hc_mult, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
layer_input = torch.empty(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
|
||||
)
|
||||
aux = torch.empty(
|
||||
num_tokens if capture_aux else 0,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
)
|
||||
if num_tokens == 0:
|
||||
return (
|
||||
torch.empty_like(residual),
|
||||
post.unsqueeze(-1),
|
||||
comb.view(num_tokens, hc_mult, hc_mult),
|
||||
layer_input,
|
||||
next_pre_mix,
|
||||
aux,
|
||||
)
|
||||
|
||||
fused_config = mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult)
|
||||
if fused_config is not None:
|
||||
mixes, sqrsum, residual_cur = _MHC_FUSED_TILELANG_KERNEL(
|
||||
comb_res_mix,
|
||||
residual,
|
||||
post_layer_mix.view(num_tokens, hc_mult),
|
||||
x,
|
||||
fn.view(mix_size, hc_mult, hidden_size),
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
mix_size,
|
||||
)
|
||||
else:
|
||||
residual_cur = torch.empty_like(residual)
|
||||
_MHC_POST_TILELANG_KERNEL(
|
||||
comb_res_mix,
|
||||
residual,
|
||||
post_layer_mix.view(num_tokens, hc_mult),
|
||||
x,
|
||||
residual_cur,
|
||||
hc_mult,
|
||||
hidden_size,
|
||||
)
|
||||
# The delayed epilogue is compiled per bucketed split count, so the
|
||||
# projection has to use the same bucket rather than the raw estimate.
|
||||
use_deep_gemm = is_deep_gemm_supported()
|
||||
n_splits = (
|
||||
compute_mhc_pre_num_splits(input_size, num_tokens) if use_deep_gemm else 1
|
||||
)
|
||||
mixes = torch.empty(
|
||||
n_splits, num_tokens, mix_size, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
sqrsum = torch.empty(
|
||||
n_splits, num_tokens, dtype=torch.float32, device=residual.device
|
||||
)
|
||||
residual_cur_2d = residual_cur.view(num_tokens, input_size)
|
||||
if use_deep_gemm:
|
||||
tf32_hc_prenorm_gemm(residual_cur_2d, fn, mixes, sqrsum, n_splits)
|
||||
else:
|
||||
_HC_PRENORM_GEMM_TILELANG_KERNEL(
|
||||
residual_cur_2d,
|
||||
fn,
|
||||
mixes,
|
||||
sqrsum,
|
||||
input_size,
|
||||
1,
|
||||
)
|
||||
|
||||
outputs = (
|
||||
residual_cur,
|
||||
post.unsqueeze(-1),
|
||||
comb.view(num_tokens, hc_mult, hc_mult),
|
||||
layer_input,
|
||||
next_pre_mix,
|
||||
aux,
|
||||
)
|
||||
if norm_weight is not None:
|
||||
assert norm_weight.shape == (hidden_size,)
|
||||
assert norm_weight.dtype == torch.bfloat16 and norm_weight.is_contiguous()
|
||||
MHC_PRE_NORM_KERNEL(
|
||||
mixes,
|
||||
sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_cur,
|
||||
post,
|
||||
comb,
|
||||
layer_input,
|
||||
norm_weight,
|
||||
pre_mix if pre_mix is not None else post,
|
||||
next_pre_mix,
|
||||
aux if capture_aux else layer_input,
|
||||
hidden_size=hidden_size,
|
||||
rms_eps=rms_eps,
|
||||
hc_pre_eps=hc_pre_eps,
|
||||
hc_sinkhorn_eps=hc_sinkhorn_eps,
|
||||
hc_post_mult_value=hc_post_mult_value,
|
||||
sinkhorn_repeat=sinkhorn_repeat,
|
||||
norm_eps=norm_eps,
|
||||
hc_mult=hc_mult,
|
||||
use_pre_mix_in=pre_mix is not None,
|
||||
save_pre_mix=True,
|
||||
rms_numel=input_size,
|
||||
write_aux=capture_aux,
|
||||
)
|
||||
return outputs
|
||||
mhc_pre_big_fuse_tilelang(
|
||||
mixes,
|
||||
sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_cur,
|
||||
post,
|
||||
comb,
|
||||
layer_input,
|
||||
pre_mix if pre_mix is not None else post,
|
||||
next_pre_mix,
|
||||
aux if capture_aux else layer_input,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
mixes.shape[0],
|
||||
hc_mult,
|
||||
use_pre_mix_in=pre_mix is not None,
|
||||
save_pre_mix=True,
|
||||
rms_numel=input_size,
|
||||
write_aux=capture_aux,
|
||||
)
|
||||
return outputs
|
||||
|
||||
|
||||
def _mhc_fused_post_pre_delayed_tilelang_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
capture_aux: bool = False,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
|
||||
]:
|
||||
num_tokens, hc_mult, hidden_size = residual.shape
|
||||
return (
|
||||
torch.empty_like(residual),
|
||||
torch.empty(
|
||||
num_tokens, hc_mult, 1, dtype=torch.float32, device=residual.device
|
||||
),
|
||||
torch.empty(
|
||||
num_tokens, hc_mult, hc_mult, dtype=torch.float32, device=residual.device
|
||||
),
|
||||
torch.empty(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=residual.device
|
||||
),
|
||||
torch.empty(num_tokens, hc_mult, dtype=torch.float32, device=residual.device),
|
||||
torch.empty(
|
||||
num_tokens if capture_aux else 0,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=residual.device,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def mhc_pre_tilelang(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
@@ -574,6 +848,7 @@ def mhc_fused_post_pre_tilelang(
|
||||
_MHC_FUSED_TILELANG_KERNEL,
|
||||
_MHC_POST_TILELANG_KERNEL,
|
||||
_MHC_PRE_BIG_FUSE_TILELANG_KERNEL,
|
||||
mhc_fused_post_pre_split_config,
|
||||
)
|
||||
|
||||
assert residual.dtype == torch.bfloat16
|
||||
@@ -614,7 +889,7 @@ def mhc_fused_post_pre_tilelang(
|
||||
post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult)
|
||||
comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult)
|
||||
|
||||
use_small_fma = num_tokens <= 16
|
||||
fused_config = mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult)
|
||||
|
||||
post_mix_cur = torch.empty(
|
||||
num_tokens,
|
||||
@@ -635,7 +910,7 @@ def mhc_fused_post_pre_tilelang(
|
||||
device=residual.device,
|
||||
)
|
||||
|
||||
if use_small_fma:
|
||||
if fused_config is not None:
|
||||
gemm_out_mul, gemm_out_sqrsum, residual_cur = _MHC_FUSED_TILELANG_KERNEL(
|
||||
comb_res_mix_flat,
|
||||
residual_flat,
|
||||
@@ -815,6 +1090,12 @@ direct_register_custom_op(
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_pre_delayed_tilelang_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_fused_post_pre_delayed_tilelang",
|
||||
op_func=mhc_fused_post_pre_delayed_tilelang,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_fused_post_pre_delayed_tilelang_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre_tilelang",
|
||||
op_func=mhc_pre_tilelang,
|
||||
|
||||
@@ -37,8 +37,67 @@ def compute_num_split(block_k: int, k: int | None, grid_size: int) -> int:
|
||||
return split_k
|
||||
|
||||
|
||||
def _mhc_fused_n_splits(num_tokens: int, hidden_size: int) -> int:
|
||||
return 8 if num_tokens < 8 and hidden_size <= 4096 else 4
|
||||
# ``mhc_fused_tilelang`` splits the hidden size across a fixed thread block, so
|
||||
# every (tile_n, n_thr) pair is one more kernel to compile. Tuned on GB300
|
||||
# (hc_mult 4, hidden_size 5120 and 7168) against the separate post + split-k
|
||||
# GEMM it replaces; the bands are coarse because the loss against a per-shape
|
||||
# optimum is ~1-2%, while one fixed config gives up to 18% at the top of the
|
||||
# range.
|
||||
_FUSED_POST_PRE_N_THR = 128
|
||||
_FUSED_POST_PRE_MAX_TOKENS = 32
|
||||
|
||||
|
||||
def mhc_fused_post_pre_split_config(
|
||||
num_tokens: int, hidden_size: int, hc_mult: int
|
||||
) -> tuple[int, int, int] | None:
|
||||
"""Pick ``(tile_n, n_splits, n_thr)`` for the fused post + pre-norm GEMM.
|
||||
|
||||
Returns None when a separate post kernel followed by a split-k GEMM is the
|
||||
better choice, or when the hidden size does not divide evenly across the
|
||||
fused kernel's block. One source of truth for the dispatch decision, the
|
||||
compile key and the launch, which must agree.
|
||||
"""
|
||||
if num_tokens > _FUSED_POST_PRE_MAX_TOKENS:
|
||||
return None
|
||||
n_thr = _FUSED_POST_PRE_N_THR
|
||||
if hidden_size % n_thr:
|
||||
return None
|
||||
# The projection tiles supply the parallelism a small token count cannot,
|
||||
# so the tile grows with the batch: 1.3x at one token and 1.03x at 32
|
||||
# against a split-k GEMM given the split its own estimator computes.
|
||||
if num_tokens < 16:
|
||||
tile_n, n_splits = 2, 8
|
||||
else:
|
||||
tile_n, n_splits = 6, 8
|
||||
# The kernel drops projection tiles and k-slices it cannot fill evenly.
|
||||
mix_size = hc_mult * (hc_mult + 2)
|
||||
while tile_n > 1 and mix_size % tile_n:
|
||||
tile_n -= 1
|
||||
while n_splits > 1 and hidden_size % (n_splits * n_thr):
|
||||
n_splits //= 2
|
||||
return tile_n, n_splits, n_thr
|
||||
|
||||
|
||||
def require_fused_post_pre_config(
|
||||
num_tokens: int, hidden_size: int, hc_mult: int
|
||||
) -> tuple[int, int, int]:
|
||||
"""The config for a shape the caller has already committed to fusing."""
|
||||
config = mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult)
|
||||
if config is None:
|
||||
raise ValueError(
|
||||
"the fused mHC post + pre-norm GEMM does not cover num_tokens="
|
||||
f"{num_tokens} at hidden_size={hidden_size}"
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def mhc_fused_post_pre_splits(hidden_size: int, hc_mult: int) -> tuple[int, ...]:
|
||||
"""Every split-k factor the fused post + pre-norm GEMM path can pick."""
|
||||
configs = (
|
||||
mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult)
|
||||
for num_tokens in range(1, _FUSED_POST_PRE_MAX_TOKENS + 1)
|
||||
)
|
||||
return tuple(sorted({config[1] for config in configs if config is not None}))
|
||||
|
||||
|
||||
@tilelang_jit
|
||||
@@ -53,6 +112,7 @@ def mhc_pre_big_fuse_tilelang(
|
||||
layer_input,
|
||||
pre_mix_in,
|
||||
pre_mix_out,
|
||||
aux_out,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
@@ -64,6 +124,7 @@ def mhc_pre_big_fuse_tilelang(
|
||||
use_pre_mix_in: bool = False,
|
||||
save_pre_mix: bool = False,
|
||||
rms_numel: int = 0,
|
||||
write_aux: bool = False,
|
||||
):
|
||||
"""Fuse coefficient generation and residual collapse after the projection.
|
||||
|
||||
@@ -88,6 +149,7 @@ def mhc_pre_big_fuse_tilelang(
|
||||
|
||||
pre_mix_in: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type]
|
||||
pre_mix_out: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type]
|
||||
aux_out: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
|
||||
|
||||
with T.Kernel(num_tokens, threads=96) as i:
|
||||
if ENABLE_PDL:
|
||||
@@ -188,11 +250,24 @@ def mhc_pre_big_fuse_tilelang(
|
||||
|
||||
ol = T.alloc_fragment(hidden_block, T.float32)
|
||||
T.clear(ol)
|
||||
if write_aux:
|
||||
# Aux consumers (draft models) take the plain stream mean
|
||||
# of the same residual this collapse reads.
|
||||
aux = T.alloc_fragment(hidden_block, T.float32)
|
||||
T.clear(aux)
|
||||
|
||||
for i_hc in T.serial(hc_mult):
|
||||
pre = pre_mix_shared[i_hc]
|
||||
for i1_h in T.Parallel(hidden_block):
|
||||
ol[i1_h] += pre * xl[i_hc, i1_h]
|
||||
if write_aux:
|
||||
aux[i1_h] += xl[i_hc, i1_h]
|
||||
|
||||
if write_aux:
|
||||
for i1_h in T.Parallel(hidden_block):
|
||||
aux_out[i, i0_h * hidden_block + i1_h] = T.bfloat16(
|
||||
aux[i1_h] / hc_mult
|
||||
)
|
||||
|
||||
T.copy(ol, layer_input[i, i0_h * hidden_block])
|
||||
|
||||
@@ -216,6 +291,7 @@ def mhc_pre_big_fuse_with_norm_tilelang(
|
||||
norm_weight,
|
||||
pre_mix_in,
|
||||
pre_mix_out,
|
||||
aux_out,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
@@ -229,6 +305,7 @@ def mhc_pre_big_fuse_with_norm_tilelang(
|
||||
use_pre_mix_in: bool = False,
|
||||
save_pre_mix: bool = False,
|
||||
rms_numel: int = 0,
|
||||
write_aux: bool = False,
|
||||
):
|
||||
num_tokens = T.dynamic("num_tokens")
|
||||
hc_mult3 = hc_mult * (2 + hc_mult)
|
||||
@@ -250,6 +327,7 @@ def mhc_pre_big_fuse_with_norm_tilelang(
|
||||
|
||||
pre_mix_in: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type]
|
||||
pre_mix_out: T.Tensor[[num_tokens, hc_mult], T.float32] # type: ignore[no-redef, valid-type]
|
||||
aux_out: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
|
||||
|
||||
with T.Kernel(num_tokens, threads=96) as i:
|
||||
rms = T.alloc_fragment(1, T.float32)
|
||||
@@ -345,11 +423,24 @@ def mhc_pre_big_fuse_with_norm_tilelang(
|
||||
|
||||
ol = T.alloc_fragment(hidden_block, T.float32)
|
||||
T.clear(ol)
|
||||
if write_aux:
|
||||
# Aux consumers (draft models) take the plain stream mean
|
||||
# of the same residual this collapse reads.
|
||||
aux = T.alloc_fragment(hidden_block, T.float32)
|
||||
T.clear(aux)
|
||||
|
||||
for i_hc in T.serial(hc_mult):
|
||||
pre = pre_mix_shared[i_hc]
|
||||
for i1_h in T.Parallel(hidden_block):
|
||||
ol[i1_h] += pre * xl[i_hc, i1_h]
|
||||
if write_aux:
|
||||
aux[i1_h] += xl[i_hc, i1_h]
|
||||
|
||||
if write_aux:
|
||||
for i1_h in T.Parallel(hidden_block):
|
||||
aux_out[i, i0_h * hidden_block + i1_h] = T.bfloat16(
|
||||
aux[i1_h] / hc_mult
|
||||
)
|
||||
|
||||
if save_pre_mix:
|
||||
# Keep the BF16 boundary before the delayed input RMSNorm.
|
||||
@@ -1211,9 +1302,17 @@ class MhcPreBigFuseTileLangKernel(
|
||||
middle_args = (residual_out, *output_args, norm_weight)
|
||||
elif compile_key.use_norm_weight:
|
||||
assert norm_weight is not None
|
||||
middle_args = (*output_args, norm_weight, post_mix, post_mix)
|
||||
# layer_input stands in for the aux buffer on both epilogues: same
|
||||
# shape and dtype, and write_aux is off here so it is never written.
|
||||
middle_args = (
|
||||
*output_args,
|
||||
norm_weight,
|
||||
post_mix,
|
||||
post_mix,
|
||||
layer_input,
|
||||
)
|
||||
else:
|
||||
middle_args = (*output_args, post_mix, post_mix)
|
||||
middle_args = (*output_args, post_mix, post_mix, layer_input)
|
||||
norm_eps_args = (compile_key.norm_eps,) if compile_key.use_norm_weight else ()
|
||||
return (
|
||||
*common_args,
|
||||
@@ -1247,8 +1346,14 @@ class MhcPreBigFuseTileLangKernel(
|
||||
if use_pre_gemm_splits
|
||||
else n_splits
|
||||
)
|
||||
fused_n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
|
||||
actual_n_splits = fused_n_splits if use_fused_tilelang else pre_gemm_n_splits
|
||||
# The epilogue reduces over whatever the fused kernel split the GEMM
|
||||
# into, so both have to read the same config.
|
||||
fused_config = mhc_fused_post_pre_split_config(num_tokens, hidden_size, hc_mult)
|
||||
actual_n_splits = (
|
||||
fused_config[1]
|
||||
if use_fused_tilelang and fused_config is not None
|
||||
else pre_gemm_n_splits
|
||||
)
|
||||
actual_norm_eps = broadcast_norm_eps if is_broadcast else norm_eps
|
||||
actual_use_norm_weight = use_norm_weight or is_broadcast
|
||||
return self.CompileKey(
|
||||
@@ -1540,6 +1645,7 @@ class MhcFusedTileLangKernel(
|
||||
hc_mult: int
|
||||
n_splits: int
|
||||
tile_n: int
|
||||
n_thr: int
|
||||
|
||||
@staticmethod
|
||||
def kernel() -> Any:
|
||||
@@ -1552,14 +1658,15 @@ class MhcFusedTileLangKernel(
|
||||
hidden_size: int,
|
||||
hc_mult: int,
|
||||
) -> CompileKey:
|
||||
# TODO(gnovack): investigate autotuning these heuristics
|
||||
tile_n = 2 if num_tokens < 8 else 3
|
||||
n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
|
||||
# Subscripts rather than an unpack: the warmup tracer parses this body
|
||||
# and allows only plain assignments before the return.
|
||||
config = require_fused_post_pre_config(num_tokens, hidden_size, hc_mult)
|
||||
return self.CompileKey(
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
n_splits=n_splits,
|
||||
tile_n=tile_n,
|
||||
n_splits=config[1],
|
||||
tile_n=config[0],
|
||||
n_thr=config[2],
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
@@ -1574,11 +1681,11 @@ class MhcFusedTileLangKernel(
|
||||
num_tokens=WarmupIntRange(1, max_tokens + 1),
|
||||
hidden_size=hidden_size,
|
||||
hc_mult=hc_mult,
|
||||
_when=lambda *, num_tokens: num_tokens <= 16,
|
||||
_when=lambda *, num_tokens: num_tokens <= _FUSED_POST_PRE_MAX_TOKENS,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
num_tokens = 1 if compile_key.tile_n == 2 else 8
|
||||
num_tokens = 1 if compile_key.tile_n == 2 else 16
|
||||
hidden_size = compile_key.hidden_size
|
||||
hc_mult = compile_key.hc_mult
|
||||
hc_mult3 = hc_mult * (2 + hc_mult)
|
||||
@@ -1617,8 +1724,9 @@ class MhcFusedTileLangKernel(
|
||||
hc_mult3: int,
|
||||
) -> TileLangLaunchSpec:
|
||||
num_tokens = residual_in.shape[0]
|
||||
tile_n = 2 if num_tokens < 8 else 3
|
||||
n_splits = _mhc_fused_n_splits(num_tokens, hidden_size)
|
||||
tile_n, n_splits, n_thr = require_fused_post_pre_config(
|
||||
num_tokens, hidden_size, hc_mult
|
||||
)
|
||||
yp_out = residual_in.new_empty(
|
||||
(n_splits, num_tokens, hc_mult3), dtype=torch.float32
|
||||
)
|
||||
@@ -1639,7 +1747,7 @@ class MhcFusedTileLangKernel(
|
||||
hidden_size,
|
||||
hc_mult3,
|
||||
),
|
||||
dict(tile_n=tile_n, split_k=n_splits),
|
||||
dict(n_thr=n_thr, tile_n=tile_n),
|
||||
(yp_out, rp_out, residual_out),
|
||||
)
|
||||
|
||||
|
||||
@@ -36,23 +36,33 @@ class MHCPreNormKernel(VllmJitKernel["MHCPreNormKernel.CompileKey"]):
|
||||
use_pre_mix_in: bool
|
||||
rms_numel: int
|
||||
save_pre_mix: bool = True
|
||||
write_aux: bool = False
|
||||
|
||||
kernel: Any = staticmethod(mhc_pre_big_fuse_with_norm_tilelang)
|
||||
|
||||
def dispatch(self, *, n_splits, **fields) -> CompileKey: # type: ignore[override]
|
||||
return self.CompileKey(n_splits=n_splits, **fields)
|
||||
|
||||
def get_warmup_keys(self, *, max_tokens: int, **fields) -> list[CompileKey]:
|
||||
# The split heuristic changes only at 64-token boundaries.
|
||||
splits = (
|
||||
sorted(
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
max_tokens: int,
|
||||
extra_splits: Iterable[int] = (),
|
||||
**fields,
|
||||
) -> list[CompileKey]:
|
||||
# The split heuristic changes only at 64-token boundaries. Callers that
|
||||
# reach this kernel through another GEMM (the fused post + pre-norm
|
||||
# GEMM) pass the splits that GEMM picks in extra_splits.
|
||||
splits = sorted(
|
||||
set(extra_splits)
|
||||
| (
|
||||
{
|
||||
compute_mhc_pre_num_splits(fields["rms_numel"], num_tokens)
|
||||
for num_tokens in range(1, max_tokens + 1, 64)
|
||||
}
|
||||
if is_deep_gemm_supported()
|
||||
else {1}
|
||||
)
|
||||
if is_deep_gemm_supported()
|
||||
else [1]
|
||||
)
|
||||
return self._trace_dispatch(self.dispatch)(n_splits=splits, **fields)
|
||||
|
||||
|
||||
@@ -207,7 +207,7 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
|
||||
residual = post_mix = res_mix = pre_mix = None
|
||||
for layer in self.layers:
|
||||
hidden_states, residual, post_mix, res_mix, pre_mix = layer(
|
||||
hidden_states, residual, post_mix, res_mix, pre_mix, _ = layer(
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.distributed import (
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
mhc_fused_post_pre_delayed_tilelang,
|
||||
mhc_post_tilelang,
|
||||
mhc_pre_delayed_tilelang,
|
||||
)
|
||||
@@ -214,17 +215,45 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
self.hc_eps = config.hc_eps
|
||||
self.hc_post_alpha = 2.0
|
||||
if vllm_config.kernel_config.enable_jit_warmup and current_platform.is_cuda():
|
||||
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
|
||||
mhc_fused_post_pre_splits,
|
||||
)
|
||||
from vllm.model_executor.kernels.mhc.warmup import MHC_PRE_NORM_KERNEL
|
||||
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
if self.use_sequence_parallel:
|
||||
tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
max_tokens = (max_tokens + tp_size - 1) // tp_size
|
||||
for input_size, use_pre_mix in (
|
||||
(self.hidden_size, False),
|
||||
(self.hc_mult * self.hidden_size, False),
|
||||
(self.hc_mult * self.hidden_size, True),
|
||||
):
|
||||
# The epilogue compiles per projection width and per pre-mix mode.
|
||||
# The first layer projects the broadcast embedding, so it reads one
|
||||
# hidden_size-wide row and selects stream zero; every later sublayer
|
||||
# projects the full hc stream, and collapses it with the pre-mix the
|
||||
# previous sublayer carried in. Those later shapes also arrive
|
||||
# through the fused post + pre-norm GEMM, which picks split-k
|
||||
# factors the token sweep below never produces.
|
||||
broadcast_embedding = {
|
||||
"rms_numel": self.hidden_size,
|
||||
"use_pre_mix_in": False,
|
||||
"extra_splits": (),
|
||||
}
|
||||
hc_stream = {
|
||||
"rms_numel": self.hc_mult * self.hidden_size,
|
||||
"extra_splits": mhc_fused_post_pre_splits(
|
||||
self.hidden_size, self.hc_mult
|
||||
),
|
||||
}
|
||||
variants = [
|
||||
broadcast_embedding,
|
||||
{**hc_stream, "use_pre_mix_in": False},
|
||||
{**hc_stream, "use_pre_mix_in": True},
|
||||
]
|
||||
if vllm_config.speculative_config is not None:
|
||||
# Draft setups read aux hidden states out of the same collapse,
|
||||
# which is its own specialization of the kernel.
|
||||
variants.append(
|
||||
{**hc_stream, "use_pre_mix_in": True, "write_aux": True}
|
||||
)
|
||||
for variant in variants:
|
||||
MHC_PRE_NORM_KERNEL.register_warmup(
|
||||
max_tokens=max_tokens,
|
||||
hidden_size=self.hidden_size,
|
||||
@@ -235,8 +264,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
sinkhorn_repeat=self.hc_sinkhorn_iters,
|
||||
norm_eps=self.rms_norm_eps,
|
||||
hc_mult=self.hc_mult,
|
||||
use_pre_mix_in=use_pre_mix,
|
||||
rms_numel=input_size,
|
||||
**variant,
|
||||
)
|
||||
mix_hc = (2 + self.hc_mult) * self.hc_mult
|
||||
hc_dim = self.hc_mult * self.hidden_size
|
||||
@@ -295,7 +323,17 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
residual: torch.Tensor | None = None,
|
||||
engram_hashes: torch.Tensor | None = None,
|
||||
engram_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
*,
|
||||
capture_previous_aux: bool = False,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor | None,
|
||||
]:
|
||||
previous_aux: torch.Tensor | None = None
|
||||
# The reference collapses each sublayer's input with the *previous*
|
||||
# sublayer's pre-mix: attention uses the pre-mix carried in (identity
|
||||
# for the first layer), the FFN uses this layer's attention pre-mix.
|
||||
@@ -335,17 +373,19 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
norm_weight=self.attn_norm.weight,
|
||||
norm_eps=self.attn_norm.variance_epsilon,
|
||||
)
|
||||
else:
|
||||
residual = mhc_post_tilelang(x, residual, post_mix, res_mix)
|
||||
if self.engram is not None and engram_hashes is not None:
|
||||
# Engram injection happens between the previous sublayer's
|
||||
# post and this block's pre, on the full hc stream, so the
|
||||
# mix coefficients see the injected stream.
|
||||
residual = self.engram(
|
||||
residual,
|
||||
engram_hashes[:, self.engram.layer_hash_index],
|
||||
engram_mask,
|
||||
)
|
||||
elif self.engram is not None and engram_hashes is not None:
|
||||
# Engram injection happens between the previous sublayer's post
|
||||
# and this block's pre, on the full hc stream, so the mix
|
||||
# coefficients see the injected stream. The injection also keeps
|
||||
# the post out of the pre-norm GEMM's fused prologue.
|
||||
previous_post = mhc_post_tilelang(x, residual, post_mix, res_mix)
|
||||
if capture_previous_aux:
|
||||
previous_aux = previous_post.mean(dim=1)
|
||||
residual = self.engram(
|
||||
previous_post,
|
||||
engram_hashes[:, self.engram.layer_hash_index],
|
||||
engram_mask,
|
||||
)
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang(
|
||||
residual,
|
||||
self.hc_attn_fn,
|
||||
@@ -360,6 +400,31 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
norm_weight=self.attn_norm.weight,
|
||||
norm_eps=self.attn_norm.variance_epsilon,
|
||||
)
|
||||
else:
|
||||
# The collapse already reads the post-mapped streams, so the mean
|
||||
# aux consumers want comes out of the same kernel.
|
||||
residual, post_mix, res_mix, x, attn_pre, aux = (
|
||||
mhc_fused_post_pre_delayed_tilelang(
|
||||
x,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
self.hc_attn_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
self.hc_eps,
|
||||
self.hc_post_alpha,
|
||||
self.hc_sinkhorn_iters,
|
||||
pre_mix=pre_mix,
|
||||
norm_weight=self.attn_norm.weight,
|
||||
norm_eps=self.attn_norm.variance_epsilon,
|
||||
capture_aux=capture_previous_aux,
|
||||
)
|
||||
)
|
||||
if capture_previous_aux:
|
||||
previous_aux = aux
|
||||
|
||||
if self.use_sequence_parallel:
|
||||
x = sp_all_gather(x)[: positions.shape[0]]
|
||||
@@ -368,23 +433,27 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
if self.use_sequence_parallel:
|
||||
x = sp_reduce_scatter(x)
|
||||
|
||||
residual = mhc_post_tilelang(x, residual, post_mix, res_mix)
|
||||
post_mix, res_mix, x, ffn_pre = mhc_pre_delayed_tilelang(
|
||||
residual,
|
||||
self.hc_ffn_fn,
|
||||
self.hc_ffn_scale,
|
||||
self.hc_ffn_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
self.hc_eps,
|
||||
self.hc_post_alpha,
|
||||
self.hc_sinkhorn_iters,
|
||||
pre_mix=attn_pre,
|
||||
norm_weight=self.ffn_norm.weight,
|
||||
norm_eps=self.ffn_norm.variance_epsilon,
|
||||
residual, post_mix, res_mix, x, ffn_pre, _ = (
|
||||
mhc_fused_post_pre_delayed_tilelang(
|
||||
x,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
self.hc_ffn_fn,
|
||||
self.hc_ffn_scale,
|
||||
self.hc_ffn_base,
|
||||
self.rms_norm_eps,
|
||||
self.hc_eps,
|
||||
self.hc_eps,
|
||||
self.hc_post_alpha,
|
||||
self.hc_sinkhorn_iters,
|
||||
pre_mix=attn_pre,
|
||||
norm_weight=self.ffn_norm.weight,
|
||||
norm_eps=self.ffn_norm.variance_epsilon,
|
||||
)
|
||||
)
|
||||
x = self.ffn(x, input_ids)
|
||||
return x, residual, post_mix, res_mix, ffn_pre
|
||||
return x, residual, post_mix, res_mix, ffn_pre, previous_aux
|
||||
|
||||
|
||||
class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
@@ -631,13 +700,14 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
if not get_pp_group().is_first_rank:
|
||||
assert intermediate_tensors is not None
|
||||
pre_mix = intermediate_tensors["pre_mix"]
|
||||
aux_hidden_states: list[torch.Tensor] = []
|
||||
final_aux_recon: torch.Tensor | None = None # avoid duplicate mhc_post call
|
||||
# Every layer's post runs inside the next layer's fused pre, so aux
|
||||
# hidden states are read back from there instead of recomputed.
|
||||
aux_hidden_by_layer: dict[int, torch.Tensor] = {}
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer),
|
||||
start=self.start_layer,
|
||||
):
|
||||
hidden_states, residual, post_mix, res_mix, pre_mix = layer(
|
||||
hidden_states, residual, post_mix, res_mix, pre_mix, previous_aux = layer(
|
||||
hidden_states,
|
||||
positions,
|
||||
input_ids,
|
||||
@@ -647,25 +717,29 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
residual,
|
||||
engram_hashes,
|
||||
engram_mask,
|
||||
capture_previous_aux=idx in self.aux_hidden_state_layers,
|
||||
)
|
||||
if idx + 1 in self.aux_hidden_state_layers:
|
||||
# Reconstruct the aux hidden state for draft models
|
||||
aux_recon = mhc_post_tilelang(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
aux_hidden_state = aux_recon.mean(dim=1)
|
||||
if previous_aux is not None:
|
||||
# idx is the one-based id of the layer whose post this is.
|
||||
if self.use_sequence_parallel:
|
||||
aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens]
|
||||
aux_hidden_states.append(aux_hidden_state)
|
||||
final_aux_recon = aux_recon
|
||||
previous_aux = sp_all_gather(previous_aux)[:full_num_tokens]
|
||||
aux_hidden_by_layer[idx] = previous_aux
|
||||
if layer is not None:
|
||||
# Reuse if the last layer was captured as an aux hidden state
|
||||
# The last layer has no successor to fold its post into.
|
||||
hidden_states = mhc_post_tilelang(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
if self.end_layer in self.aux_hidden_state_layers:
|
||||
hidden_states = final_aux_recon
|
||||
else:
|
||||
hidden_states = mhc_post_tilelang(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
final_aux = hidden_states.mean(dim=1)
|
||||
if self.use_sequence_parallel:
|
||||
final_aux = sp_all_gather(final_aux)[:full_num_tokens]
|
||||
aux_hidden_by_layer[self.end_layer] = final_aux
|
||||
|
||||
aux_hidden_states = [
|
||||
aux_hidden_by_layer[layer_id]
|
||||
for layer_id in self.aux_hidden_state_layers
|
||||
if layer_id in aux_hidden_by_layer
|
||||
]
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
|
||||
Reference in New Issue
Block a user