[Perf][DSV4.1] Overlap mHC coefficients for small TP batches (#57603)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Woosuk Kwon
2026-09-20 17:33:17 -07:00
committed by GitHub
co-authored by Codex
parent 17e50b9b76
commit 9b49f92344
6 changed files with 357 additions and 46 deletions
+129 -12
View File
@@ -266,6 +266,77 @@ def test_deepseek_v41_mhc_pre_delayed(
torch.testing.assert_close(actual[2], expected[2], atol=1.6e-2, rtol=1e-2)
@pytest.mark.skipif(
not HAS_TILELANG_MHC or not current_platform.is_device_capability_family(100),
reason="SM100 TileLang mHC required",
)
@pytest.mark.parametrize("num_tokens", [0, 1, 8, 9, 32, 128, 256])
@pytest.mark.parametrize("entry", ["broadcast", "identity", "carried"])
@pytest.mark.parametrize("hidden_size,hc_mult", [(4096, 2), (5120, 4), (7168, 4)])
def test_deepseek_v41_mhc_overlap_preserves_outputs_on_replay(
num_tokens, entry, hidden_size, hc_mult
):
"""The input is usable before joining; coefficients stay exact on graph replay."""
from vllm.models.deepseek_v41.nvidia.ops.mhc import mhc_pre_delayed_overlap
from vllm.utils.deep_gemm import is_deep_gemm_supported
if not is_deep_gemm_supported():
pytest.skip("DeepGEMM required")
set_random_seed(42)
mix_size = hc_mult * (hc_mult + 2)
residual = torch.randn(
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=DEVICE
)
fn = torch.randn(mix_size, hc_mult, hidden_size, device=DEVICE) * 0.02
x = None
pre_mix = None
if entry == "broadcast":
x = residual[:, 0].contiguous()
residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous()
fn = fn.sum(1)
else:
fn = fn.flatten(1)
if entry == "carried":
pre_mix = torch.rand(num_tokens, hc_mult, device=DEVICE)
scale = torch.tensor([0.5, 0.25, 1.0], device=DEVICE)
base = torch.randn(mix_size, device=DEVICE)
weight = torch.empty(hidden_size, dtype=torch.bfloat16, device=DEVICE).uniform_(
0.5, 1.5
)
args = (residual, fn, scale, base, 1e-20, 1e-6, 1e-6, 2.0, 20)
kwargs = dict(pre_mix=pre_mix, x=x, norm_weight=weight, norm_eps=1e-20)
side = torch.cuda.Stream()
def run():
outputs = mhc_pre_delayed_overlap(*args, **kwargs, stream=side)
consumed_input = outputs[2].clone()
torch.cuda.current_stream().wait_stream(side)
return (*outputs, consumed_input)
def check(actual):
expected = mhc_pre_delayed_tilelang(*args, **kwargs)
for a, b in zip(actual[:4], expected, strict=True):
torch.testing.assert_close(a, b, atol=0, rtol=0)
torch.testing.assert_close(actual[4], expected[2], atol=0, rtol=0)
check(run())
if num_tokens == 0:
return
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = run()
for _ in range(5):
if x is not None:
x.normal_()
residual.copy_(x.unsqueeze(1).expand_as(residual))
else:
residual.normal_()
if pre_mix is not None:
pre_mix.uniform_()
graph.replay()
check(captured)
def mhc_fused_post_pre_delayed_ref(
x: torch.Tensor,
residual: torch.Tensor,
@@ -433,10 +504,16 @@ def test_mhc_fused_post_pre_delayed_custom_op_supports_compile(carried, capture_
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.parametrize("entry", ["broadcast", "pipeline", "residual", "engram"])
@pytest.mark.parametrize(
"mhc_mode", ["disabled", "overlap", "piecewise", "large_batch"]
)
def test_deepseek_v41_decoder_mixes_match_torch(
entry, monkeypatch, default_vllm_config
entry, mhc_mode, monkeypatch, default_vllm_config
):
"""Preserve carried pre-mixes and Engram ordering at every decoder entry."""
from vllm.models.deepseek_v41.nvidia.ops.mhc import MHC_OVERLAP_MAX_TOKENS
num_tokens = MHC_OVERLAP_MAX_TOKENS + 1 if mhc_mode == "large_batch" else 3
set_random_seed(0)
decoder = DeepseekV41DecoderLayer.__new__(DeepseekV41DecoderLayer)
nn.Module.__init__(decoder)
@@ -446,6 +523,29 @@ def test_deepseek_v41_decoder_mixes_match_torch(
decoder.rms_norm_eps = 1e-20
decoder.hc_post_alpha = 2.0
decoder.use_sequence_parallel = False
decoder.mhc_stream = None
if mhc_mode != "disabled":
from vllm.utils.deep_gemm import is_deep_gemm_supported
if (
not current_platform.is_device_capability_family(100)
or not is_deep_gemm_supported()
):
pytest.skip("SM100 DeepGEMM required for overlap")
decoder.mhc_stream = torch.cuda.Stream()
if mhc_mode in ("piecewise", "large_batch"):
def unexpected_overlap(*args, **kwargs):
pytest.fail("unsupported execution must retain native mHC")
monkeypatch.setattr(
"vllm.models.deepseek_v41.nvidia.model.mhc_pre_delayed_overlap",
unexpected_overlap,
)
monkeypatch.setattr(
"vllm.models.deepseek_v41.nvidia.ops.mhc.mhc_pre_delayed_overlap",
unexpected_overlap,
)
decoder.engram = None
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -461,18 +561,18 @@ def test_deepseek_v41_decoder_mixes_match_torch(
decoder.hc_attn_scale = decoder.hc_ffn_scale = torch.ones(3)
decoder.hc_attn_base = torch.randn(24)
decoder.hc_ffn_base = torch.randn(24)
x = torch.randn(3, 5120, dtype=torch.bfloat16)
positions = torch.arange(3)
x = torch.randn(num_tokens, 5120, dtype=torch.bfloat16)
positions = torch.arange(num_tokens)
kwargs = {}
if entry != "broadcast":
kwargs["pre_mix"] = torch.rand(3, 4)
kwargs["pre_mix"] = torch.rand(num_tokens, 4)
if entry == "pipeline":
x = torch.randn(3, 4, 5120, dtype=torch.bfloat16)
x = torch.randn(num_tokens, 4, 5120, dtype=torch.bfloat16)
if entry in ("residual", "engram"):
kwargs.update(
residual=torch.randn(3, 4, 5120, dtype=torch.bfloat16),
post_mix=torch.rand(3, 4, 1),
res_mix=torch.rand(3, 4, 4),
residual=torch.randn(num_tokens, 4, 5120, dtype=torch.bfloat16),
post_mix=torch.rand(num_tokens, 4, 1),
res_mix=torch.rand(num_tokens, 4, 4),
)
if entry == "engram":
@@ -483,15 +583,22 @@ def test_deepseek_v41_decoder_mixes_match_torch(
return residual + 0.125
decoder.engram = FakeEngram()
kwargs["engram_hashes"] = torch.zeros(3, 1, 1, dtype=torch.int32)
kwargs["engram_hashes"] = torch.zeros(num_tokens, 1, 1, dtype=torch.int32)
actual = decoder(x, positions, None, **kwargs)
from vllm.config import CUDAGraphMode
from vllm.forward_context import set_forward_context
mode = CUDAGraphMode.PIECEWISE if mhc_mode == "piecewise" else CUDAGraphMode.NONE
with set_forward_context(None, default_vllm_config, cudagraph_runtime_mode=mode):
actual = decoder(x, positions, None, **kwargs)
def reference(*args, norm_weight, norm_eps, **kwargs):
post, res, collapsed, pre = mhc_pre_delayed_torch(*args, **kwargs)
return post, res, decoder.attn_norm(collapsed), pre
def fused_reference(x, residual, post_mix, res_mix, *args, capture_aux=False, **kw):
def fused_reference(
x, residual, post_mix, res_mix, *args, capture_aux=False, stream=None, **kw
):
residual = mhc_post_torch(x, residual, post_mix, res_mix)
aux = residual.mean(dim=1) if capture_aux else residual.new_empty(0)
return residual, *reference(residual, *args, **kw), aux
@@ -500,6 +607,7 @@ def test_deepseek_v41_decoder_mixes_match_torch(
"vllm.models.deepseek_v41.nvidia.model.mhc_pre_delayed_tilelang",
reference,
)
decoder.mhc_stream = None
monkeypatch.setattr(
"vllm.models.deepseek_v41.nvidia.model.mhc_shifted_post_pre",
fused_reference,
@@ -530,6 +638,7 @@ def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_conf
decoder.rms_norm_eps = 1e-6
decoder.hc_post_alpha = 2.0
decoder.use_sequence_parallel = False
decoder.mhc_stream = None
decoder.engram = None
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -772,7 +881,8 @@ def hc_head_ref(
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 128])
@pytest.mark.parametrize("hidden_size", [4096, 7168])
@pytest.mark.parametrize("hc_mult", [4])
def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult):
@pytest.mark.parametrize("fused_norm", [False, True])
def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult, fused_norm):
torch.set_default_device(DEVICE)
set_random_seed(0)
@@ -802,6 +912,12 @@ def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult):
hc_post_alpha,
sinkhorn_repeat,
)
norm_kwargs = {}
if fused_norm:
# Non-shifted mHC must still use its freshly computed pre-mix.
weight = torch.ones(hidden_size, dtype=torch.bfloat16)
norm_kwargs = dict(norm_weight=weight, norm_eps=rms_eps)
ref = (*ref[:2], F.rms_norm(ref[2], (hidden_size,), weight, rms_eps))
out = torch.ops.vllm.mhc_pre_tilelang(
residual,
fn,
@@ -812,6 +928,7 @@ def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult):
hc_sinkhorn_eps,
hc_post_alpha,
sinkhorn_repeat,
**norm_kwargs,
)
for actual, expected in zip(out, ref, strict=True):
@@ -305,8 +305,12 @@ def mhc_pre_big_fuse_with_norm_tilelang(
use_pre_mix_in: bool = False,
save_pre_mix: bool = False,
rms_numel: int = 0,
split_mode: str = "fused",
write_aux: bool = False,
):
# Split modes require shifted mHC: input collapse uses a carried pre-mix.
assert split_mode in ("fused", "stats", "input")
assert split_mode == "fused" or save_pre_mix
num_tokens = T.dynamic("num_tokens")
hc_mult3 = hc_mult * (2 + hc_mult)
if gemm_last_dim < 0:
@@ -330,26 +334,26 @@ def mhc_pre_big_fuse_with_norm_tilelang(
aux_out: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
with T.Kernel(num_tokens, threads=96) as i:
rms = T.alloc_fragment(1, T.float32)
mixes = T.alloc_fragment(hc_mult3, T.float32)
T.clear(mixes)
rms[0] = 0
if ENABLE_PDL:
T.pdl_sync()
for i_split in T.serial(n_splits):
rms[0] += gemm_out_sqrsum[i_split, i]
rms[0] = T.rsqrt(rms[0] / rms_numel + rms_eps)
for j in T.Parallel(hc_mult3):
mixes[j] = 0
for i_split in T.serial(n_splits):
mixes[j] += gemm_out_mul[i_split, i, j]
mixes[j] *= rms[0]
mixes_shared = T.alloc_shared(hc_mult3, T.float32)
T.copy(mixes, mixes_shared)
if split_mode != "input":
rms = T.alloc_fragment(1, T.float32)
mixes = T.alloc_fragment(hc_mult3, T.float32)
T.clear(mixes)
rms[0] = 0
if T.get_thread_binding() < 32:
for i_split in T.serial(n_splits):
rms[0] += gemm_out_sqrsum[i_split, i]
rms[0] = T.rsqrt(rms[0] / rms_numel + rms_eps)
for j in T.Parallel(hc_mult3):
mixes[j] = 0
for i_split in T.serial(n_splits):
mixes[j] += gemm_out_mul[i_split, i, j]
mixes[j] *= rms[0]
T.copy(mixes, mixes_shared)
if split_mode != "input" and T.get_thread_binding() < 32:
cm = T.alloc_fragment((hc_mult, hc_mult), T.float32)
for j in T.Parallel(hc_mult):
if save_pre_mix:
@@ -395,7 +399,7 @@ def mhc_pre_big_fuse_with_norm_tilelang(
for j, k in T.Parallel(hc_mult, hc_mult):
comb_mix[i, j * hc_mult + k] = cm[j, k]
else:
elif split_mode != "stats" and T.get_thread_binding() >= 32:
pre_mix_shared = T.alloc_shared(hc_mult, T.float32)
for j in T.Parallel(hc_mult):
if use_pre_mix_in:
@@ -36,6 +36,7 @@ class MHCPreNormKernel(VllmJitKernel["MHCPreNormKernel.CompileKey"]):
use_pre_mix_in: bool
rms_numel: int
save_pre_mix: bool = True
split_mode: str = "fused"
write_aux: bool = False
kernel: Any = staticmethod(mhc_pre_big_fuse_with_norm_tilelang)
+60 -16
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import typing
from collections.abc import Callable, Iterable
from functools import partial
from itertools import islice
import regex as re
@@ -17,7 +18,11 @@ from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.forward_context import (
get_forward_context,
in_piecewise_cudagraph,
is_forward_context_available,
)
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
mhc_post_tilelang,
@@ -84,6 +89,11 @@ from ..common.engram import EngramLayout, NgramHashState
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
from .engram import Engram, gather_engram_hashes
from .ops.mega_mhc import mhc_shifted_post_pre
from .ops.mhc import (
MHC_OVERLAP_MAX_TOKENS,
mhc_pre_delayed_overlap,
supports_mhc_overlap,
)
if typing.TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
@@ -188,11 +198,13 @@ class DeepseekV4DecoderLayer(nn.Module):
aux_stream_list: list[torch.cuda.Stream] | None = None,
candidate_block_buffer: torch.Tensor | None = None,
engram_layout: EngramLayout | None = None,
mhc_stream: torch.cuda.Stream | None = None,
):
super().__init__()
config = vllm_config.model_config.hf_config
self.hidden_size = config.hidden_size
self.mhc_stream = mhc_stream
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
self.engram: Engram | None = None
@@ -269,19 +281,30 @@ class DeepseekV4DecoderLayer(nn.Module):
variants.append(
{**hc_stream, "use_pre_mix_in": True, "write_aux": True}
)
# Shifted mHC splits the epilogue in two: the input collapse on the
# caller stream and coefficient generation on a side stream.
split_modes = (
("fused", "stats", "input") if mhc_stream is not None else ("fused",)
)
for variant in variants:
MHC_PRE_NORM_KERNEL.register_warmup(
max_tokens=max_tokens,
hidden_size=self.hidden_size,
rms_eps=self.rms_norm_eps,
hc_pre_eps=self.hc_eps,
hc_sinkhorn_eps=self.hc_eps,
hc_post_mult_value=self.hc_post_alpha,
sinkhorn_repeat=self.hc_sinkhorn_iters,
norm_eps=self.rms_norm_eps,
hc_mult=self.hc_mult,
**variant,
)
for split_mode in split_modes:
MHC_PRE_NORM_KERNEL.register_warmup(
split_mode=split_mode,
max_tokens=(
max_tokens
if split_mode == "fused"
else min(max_tokens, MHC_OVERLAP_MAX_TOKENS)
),
hidden_size=self.hidden_size,
rms_eps=self.rms_norm_eps,
hc_pre_eps=self.hc_eps,
hc_sinkhorn_eps=self.hc_eps,
hc_post_mult_value=self.hc_post_alpha,
sinkhorn_repeat=self.hc_sinkhorn_iters,
norm_eps=self.rms_norm_eps,
hc_mult=self.hc_mult,
**variant,
)
mix_hc = (2 + self.hc_mult) * self.hc_mult
hc_dim = self.hc_mult * self.hidden_size
self.hc_attn_fn = nn.Parameter(
@@ -351,6 +374,18 @@ class DeepseekV4DecoderLayer(nn.Module):
torch.Tensor | None,
]:
previous_aux: torch.Tensor | None = None
mhc_stream = self.mhc_stream
if mhc_stream is not None and (
in_piecewise_cudagraph()
or not 0 < positions.shape[0] <= MHC_OVERLAP_MAX_TOKENS
):
# A side stream cannot remain unjoined across breakable graph segments.
mhc_stream = None
mhc_pre = (
partial(mhc_pre_delayed_overlap, stream=mhc_stream)
if mhc_stream is not None
else mhc_pre_delayed_tilelang
)
# The reference collapses each sublayer's input with the *previous*
# sublayer's pre-mix: attention uses the pre-mix carried in (identity
# for the first layer), the FFN uses this layer's attention pre-mix.
@@ -360,7 +395,7 @@ class DeepseekV4DecoderLayer(nn.Module):
# copies and the identity pre-mix selects copy 0.
assert self.hc_attn_fn_broadcast is not None
residual = x.unsqueeze(1).expand(-1, self.hc_mult, -1).contiguous()
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang(
post_mix, res_mix, x, attn_pre = mhc_pre(
residual,
self.hc_attn_fn_broadcast,
self.hc_attn_scale,
@@ -376,7 +411,7 @@ class DeepseekV4DecoderLayer(nn.Module):
)
else:
residual = x
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang(
post_mix, res_mix, x, attn_pre = mhc_pre(
residual,
self.hc_attn_fn,
self.hc_attn_scale,
@@ -403,7 +438,7 @@ class DeepseekV4DecoderLayer(nn.Module):
engram_hashes[:, self.engram.layer_hash_index],
engram_mask,
)
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang(
post_mix, res_mix, x, attn_pre = mhc_pre(
residual,
self.hc_attn_fn,
self.hc_attn_scale,
@@ -437,6 +472,7 @@ class DeepseekV4DecoderLayer(nn.Module):
norm_weight=self.attn_norm.weight,
norm_eps=self.attn_norm.variance_epsilon,
capture_aux=capture_previous_aux,
stream=mhc_stream,
)
if capture_previous_aux:
previous_aux = aux
@@ -448,6 +484,8 @@ class DeepseekV4DecoderLayer(nn.Module):
if self.use_sequence_parallel:
x = sp_reduce_scatter(x)
if mhc_stream is not None:
torch.cuda.current_stream().wait_stream(mhc_stream)
residual, post_mix, res_mix, x, ffn_pre, _ = mhc_shifted_post_pre(
x,
residual,
@@ -464,8 +502,11 @@ class DeepseekV4DecoderLayer(nn.Module):
pre_mix=attn_pre,
norm_weight=self.ffn_norm.weight,
norm_eps=self.ffn_norm.variance_epsilon,
stream=mhc_stream,
)
x = self.ffn(x, input_ids, mega_gate_metadata)
if mhc_stream is not None:
torch.cuda.current_stream().wait_stream(mhc_stream)
return x, residual, post_mix, res_mix, ffn_pre, previous_aux
@@ -497,6 +538,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
# (compressor kv_score, indexer.weights_proj). fused_wqa_wkv stays on
# the default stream.
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
# Keep mHC independent of the streams used inside attention.
mhc_stream = torch.cuda.Stream() if supports_mhc_overlap(vllm_config) else None
# Reserved topk indices buffer for all Indexer layers to reuse.
self.topk_indices_buffer = torch.empty(
@@ -541,6 +584,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
aux_stream_list=aux_stream_list,
candidate_block_buffer=self.candidate_block_buffer,
engram_layout=self.engram_layout,
mhc_stream=mhc_stream,
),
prefix=f"{prefix}.layers",
)
@@ -7,6 +7,7 @@ import torch
from vllm.model_executor.kernels.mhc.tilelang import (
mhc_fused_post_pre_delayed_tilelang,
mhc_post_tilelang,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
@@ -106,10 +107,37 @@ def mhc_shifted_post_pre(
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
capture_aux: bool = False,
*,
stream: torch.cuda.Stream | None = None,
) -> tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
]:
"""Use Mega-mHC for carried mixing, retaining TileLang for aux capture."""
"""Dispatch shifted post/pre to overlap, Mega-mHC, or fused TileLang.
When stream is supplied, join it before consuming the returned coefficients.
"""
if stream is not None:
from .mhc import mhc_pre_delayed_overlap
residual = mhc_post_tilelang(x, residual, post_layer_mix, comb_res_mix)
aux = residual.mean(dim=1) if capture_aux else x.new_empty(0, x.shape[1])
pre_outputs = mhc_pre_delayed_overlap(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix=pre_mix,
norm_weight=norm_weight,
norm_eps=norm_eps,
stream=stream,
)
return residual, *pre_outputs, aux
if (
pre_mix is not None
and norm_weight is not None
+117
View File
@@ -0,0 +1,117 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Split shifted mHC pre so coefficient generation can overlap the sublayer."""
from functools import partial
import torch
from vllm.config import VllmConfig
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
# GB200 TP4/FlashInfer improves through 16 tokens; larger screens tie or regress.
MHC_OVERLAP_MAX_TOKENS = 16
def supports_mhc_overlap(vllm_config: VllmConfig) -> bool:
"""Check kernel requirements and safety of sharing the coefficient stream."""
config = vllm_config.model_config.hf_config
# DeepGEMM's prenorm kernel requires K % 64 == 0, N % 8 == 0, and N <= 32.
mix_size = config.hc_mult * (config.hc_mult + 2)
return (
current_platform.is_device_capability_family(100)
and is_deep_gemm_supported()
and config.hidden_size % 64 == 0
and 0 < mix_size <= 32
and mix_size % 8 == 0
and not vllm_config.parallel_config.use_ubatching
)
def mhc_pre_delayed_overlap(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
x: torch.Tensor | None = None,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
*,
stream: torch.cuda.Stream,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Prepare the input on the caller stream and coefficients on another stream.
Returns post mix, residual mix, normalized input, and next pre-mix. Only the
input is ready on the caller stream; join stream before using coefficients.
The caller must retain the weights and coefficient outputs until the join.
"""
from vllm.model_executor.kernels.mhc.warmup import (
MHC_PRE_NORM_KERNEL,
compute_mhc_pre_num_splits,
)
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
n, hc, hidden = residual.shape
assert residual.is_contiguous() and residual.dtype == torch.bfloat16
assert norm_weight is not None
if x is None:
x = residual.view(n, hc * hidden)
assert x.is_contiguous()
post = torch.empty((n, hc), device=residual.device, dtype=torch.float32)
comb = torch.empty((n, hc * hc), device=residual.device, dtype=torch.float32)
next_pre = torch.empty_like(post)
layer_input = torch.empty((n, hidden), device=residual.device, dtype=torch.bfloat16)
outputs = post.unsqueeze(-1), comb.view(n, hc, hc), layer_input, next_pre
if n == 0:
return outputs
splits = compute_mhc_pre_num_splits(x.shape[1], n)
mix = torch.empty(
(splits, n, hc * (hc + 2)), device=residual.device, dtype=torch.float32
)
sqr = torch.empty((splits, n), device=residual.device, dtype=torch.float32)
epilogue = partial(
MHC_PRE_NORM_KERNEL,
mix,
sqr,
hc_scale,
hc_base,
residual,
post,
comb,
layer_input,
norm_weight,
pre_mix if pre_mix is not None else post,
next_pre,
layer_input, # Unused aux output in split modes.
hidden_size=hidden,
rms_eps=rms_eps,
hc_pre_eps=hc_pre_eps,
hc_sinkhorn_eps=hc_sinkhorn_eps,
hc_post_mult_value=hc_post_mult_value,
sinkhorn_repeat=sinkhorn_repeat,
norm_eps=norm_eps,
hc_mult=hc,
use_pre_mix_in=pre_mix is not None,
save_pre_mix=True,
rms_numel=x.shape[1],
)
main = torch.cuda.current_stream()
# Prioritize input readiness for small batches before releasing statistics.
if n <= 8:
epilogue(split_mode="input")
stream.wait_stream(main)
with torch.cuda.stream(stream):
tf32_hc_prenorm_gemm(x, fn, mix, sqr, splits)
epilogue(split_mode="stats")
for tensor in (x, mix, sqr):
tensor.record_stream(stream)
if n > 8:
epilogue(split_mode="input")
return outputs