From 9b49f92344312c41ad61e05282c8e6a2d9bafb7f Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Sun, 20 Sep 2026 17:33:17 -0700 Subject: [PATCH] [Perf][DSV4.1] Overlap mHC coefficients for small TP batches (#57603) Signed-off-by: Woosuk Kwon Co-authored-by: Codex --- tests/kernels/test_mhc_kernels.py | 141 ++++++++++++++++-- .../kernels/mhc/tilelang_kernels.py | 38 ++--- vllm/model_executor/kernels/mhc/warmup.py | 1 + vllm/models/deepseek_v41/nvidia/model.py | 76 ++++++++-- .../deepseek_v41/nvidia/ops/mega_mhc.py | 30 +++- vllm/models/deepseek_v41/nvidia/ops/mhc.py | 117 +++++++++++++++ 6 files changed, 357 insertions(+), 46 deletions(-) create mode 100644 vllm/models/deepseek_v41/nvidia/ops/mhc.py diff --git a/tests/kernels/test_mhc_kernels.py b/tests/kernels/test_mhc_kernels.py index 08b1a24c85..8f7df55602 100644 --- a/tests/kernels/test_mhc_kernels.py +++ b/tests/kernels/test_mhc_kernels.py @@ -266,6 +266,77 @@ def test_deepseek_v41_mhc_pre_delayed( torch.testing.assert_close(actual[2], expected[2], atol=1.6e-2, rtol=1e-2) +@pytest.mark.skipif( + not HAS_TILELANG_MHC or not current_platform.is_device_capability_family(100), + reason="SM100 TileLang mHC required", +) +@pytest.mark.parametrize("num_tokens", [0, 1, 8, 9, 32, 128, 256]) +@pytest.mark.parametrize("entry", ["broadcast", "identity", "carried"]) +@pytest.mark.parametrize("hidden_size,hc_mult", [(4096, 2), (5120, 4), (7168, 4)]) +def test_deepseek_v41_mhc_overlap_preserves_outputs_on_replay( + num_tokens, entry, hidden_size, hc_mult +): + """The input is usable before joining; coefficients stay exact on graph replay.""" + from vllm.models.deepseek_v41.nvidia.ops.mhc import mhc_pre_delayed_overlap + from vllm.utils.deep_gemm import is_deep_gemm_supported + + if not is_deep_gemm_supported(): + pytest.skip("DeepGEMM required") + set_random_seed(42) + mix_size = hc_mult * (hc_mult + 2) + residual = torch.randn( + num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=DEVICE + ) + fn = torch.randn(mix_size, hc_mult, hidden_size, device=DEVICE) * 0.02 + x = None + pre_mix = None + if entry == "broadcast": + x = residual[:, 0].contiguous() + residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous() + fn = fn.sum(1) + else: + fn = fn.flatten(1) + if entry == "carried": + pre_mix = torch.rand(num_tokens, hc_mult, device=DEVICE) + scale = torch.tensor([0.5, 0.25, 1.0], device=DEVICE) + base = torch.randn(mix_size, device=DEVICE) + weight = torch.empty(hidden_size, dtype=torch.bfloat16, device=DEVICE).uniform_( + 0.5, 1.5 + ) + args = (residual, fn, scale, base, 1e-20, 1e-6, 1e-6, 2.0, 20) + kwargs = dict(pre_mix=pre_mix, x=x, norm_weight=weight, norm_eps=1e-20) + side = torch.cuda.Stream() + + def run(): + outputs = mhc_pre_delayed_overlap(*args, **kwargs, stream=side) + consumed_input = outputs[2].clone() + torch.cuda.current_stream().wait_stream(side) + return (*outputs, consumed_input) + + def check(actual): + expected = mhc_pre_delayed_tilelang(*args, **kwargs) + for a, b in zip(actual[:4], expected, strict=True): + torch.testing.assert_close(a, b, atol=0, rtol=0) + torch.testing.assert_close(actual[4], expected[2], atol=0, rtol=0) + + check(run()) + if num_tokens == 0: + return + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = run() + for _ in range(5): + if x is not None: + x.normal_() + residual.copy_(x.unsqueeze(1).expand_as(residual)) + else: + residual.normal_() + if pre_mix is not None: + pre_mix.uniform_() + graph.replay() + check(captured) + + def mhc_fused_post_pre_delayed_ref( x: torch.Tensor, residual: torch.Tensor, @@ -433,10 +504,16 @@ def test_mhc_fused_post_pre_delayed_custom_op_supports_compile(carried, capture_ @pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required") @pytest.mark.parametrize("entry", ["broadcast", "pipeline", "residual", "engram"]) +@pytest.mark.parametrize( + "mhc_mode", ["disabled", "overlap", "piecewise", "large_batch"] +) def test_deepseek_v41_decoder_mixes_match_torch( - entry, monkeypatch, default_vllm_config + entry, mhc_mode, monkeypatch, default_vllm_config ): """Preserve carried pre-mixes and Engram ordering at every decoder entry.""" + from vllm.models.deepseek_v41.nvidia.ops.mhc import MHC_OVERLAP_MAX_TOKENS + + num_tokens = MHC_OVERLAP_MAX_TOKENS + 1 if mhc_mode == "large_batch" else 3 set_random_seed(0) decoder = DeepseekV41DecoderLayer.__new__(DeepseekV41DecoderLayer) nn.Module.__init__(decoder) @@ -446,6 +523,29 @@ def test_deepseek_v41_decoder_mixes_match_torch( decoder.rms_norm_eps = 1e-20 decoder.hc_post_alpha = 2.0 decoder.use_sequence_parallel = False + decoder.mhc_stream = None + if mhc_mode != "disabled": + from vllm.utils.deep_gemm import is_deep_gemm_supported + + if ( + not current_platform.is_device_capability_family(100) + or not is_deep_gemm_supported() + ): + pytest.skip("SM100 DeepGEMM required for overlap") + decoder.mhc_stream = torch.cuda.Stream() + if mhc_mode in ("piecewise", "large_batch"): + + def unexpected_overlap(*args, **kwargs): + pytest.fail("unsupported execution must retain native mHC") + + monkeypatch.setattr( + "vllm.models.deepseek_v41.nvidia.model.mhc_pre_delayed_overlap", + unexpected_overlap, + ) + monkeypatch.setattr( + "vllm.models.deepseek_v41.nvidia.ops.mhc.mhc_pre_delayed_overlap", + unexpected_overlap, + ) decoder.engram = None from vllm.model_executor.layers.layernorm import RMSNorm @@ -461,18 +561,18 @@ def test_deepseek_v41_decoder_mixes_match_torch( decoder.hc_attn_scale = decoder.hc_ffn_scale = torch.ones(3) decoder.hc_attn_base = torch.randn(24) decoder.hc_ffn_base = torch.randn(24) - x = torch.randn(3, 5120, dtype=torch.bfloat16) - positions = torch.arange(3) + x = torch.randn(num_tokens, 5120, dtype=torch.bfloat16) + positions = torch.arange(num_tokens) kwargs = {} if entry != "broadcast": - kwargs["pre_mix"] = torch.rand(3, 4) + kwargs["pre_mix"] = torch.rand(num_tokens, 4) if entry == "pipeline": - x = torch.randn(3, 4, 5120, dtype=torch.bfloat16) + x = torch.randn(num_tokens, 4, 5120, dtype=torch.bfloat16) if entry in ("residual", "engram"): kwargs.update( - residual=torch.randn(3, 4, 5120, dtype=torch.bfloat16), - post_mix=torch.rand(3, 4, 1), - res_mix=torch.rand(3, 4, 4), + residual=torch.randn(num_tokens, 4, 5120, dtype=torch.bfloat16), + post_mix=torch.rand(num_tokens, 4, 1), + res_mix=torch.rand(num_tokens, 4, 4), ) if entry == "engram": @@ -483,15 +583,22 @@ def test_deepseek_v41_decoder_mixes_match_torch( return residual + 0.125 decoder.engram = FakeEngram() - kwargs["engram_hashes"] = torch.zeros(3, 1, 1, dtype=torch.int32) + kwargs["engram_hashes"] = torch.zeros(num_tokens, 1, 1, dtype=torch.int32) - actual = decoder(x, positions, None, **kwargs) + from vllm.config import CUDAGraphMode + from vllm.forward_context import set_forward_context + + mode = CUDAGraphMode.PIECEWISE if mhc_mode == "piecewise" else CUDAGraphMode.NONE + with set_forward_context(None, default_vllm_config, cudagraph_runtime_mode=mode): + actual = decoder(x, positions, None, **kwargs) def reference(*args, norm_weight, norm_eps, **kwargs): post, res, collapsed, pre = mhc_pre_delayed_torch(*args, **kwargs) return post, res, decoder.attn_norm(collapsed), pre - def fused_reference(x, residual, post_mix, res_mix, *args, capture_aux=False, **kw): + def fused_reference( + x, residual, post_mix, res_mix, *args, capture_aux=False, stream=None, **kw + ): residual = mhc_post_torch(x, residual, post_mix, res_mix) aux = residual.mean(dim=1) if capture_aux else residual.new_empty(0) return residual, *reference(residual, *args, **kw), aux @@ -500,6 +607,7 @@ def test_deepseek_v41_decoder_mixes_match_torch( "vllm.models.deepseek_v41.nvidia.model.mhc_pre_delayed_tilelang", reference, ) + decoder.mhc_stream = None monkeypatch.setattr( "vllm.models.deepseek_v41.nvidia.model.mhc_shifted_post_pre", fused_reference, @@ -530,6 +638,7 @@ def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_conf decoder.rms_norm_eps = 1e-6 decoder.hc_post_alpha = 2.0 decoder.use_sequence_parallel = False + decoder.mhc_stream = None decoder.engram = None from vllm.model_executor.layers.layernorm import RMSNorm @@ -772,7 +881,8 @@ def hc_head_ref( @pytest.mark.parametrize("num_tokens", [1, 4, 8, 128]) @pytest.mark.parametrize("hidden_size", [4096, 7168]) @pytest.mark.parametrize("hc_mult", [4]) -def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult): +@pytest.mark.parametrize("fused_norm", [False, True]) +def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult, fused_norm): torch.set_default_device(DEVICE) set_random_seed(0) @@ -802,6 +912,12 @@ def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult): hc_post_alpha, sinkhorn_repeat, ) + norm_kwargs = {} + if fused_norm: + # Non-shifted mHC must still use its freshly computed pre-mix. + weight = torch.ones(hidden_size, dtype=torch.bfloat16) + norm_kwargs = dict(norm_weight=weight, norm_eps=rms_eps) + ref = (*ref[:2], F.rms_norm(ref[2], (hidden_size,), weight, rms_eps)) out = torch.ops.vllm.mhc_pre_tilelang( residual, fn, @@ -812,6 +928,7 @@ def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult): hc_sinkhorn_eps, hc_post_alpha, sinkhorn_repeat, + **norm_kwargs, ) for actual, expected in zip(out, ref, strict=True): diff --git a/vllm/model_executor/kernels/mhc/tilelang_kernels.py b/vllm/model_executor/kernels/mhc/tilelang_kernels.py index fa2d5e1700..33de3b9ce5 100644 --- a/vllm/model_executor/kernels/mhc/tilelang_kernels.py +++ b/vllm/model_executor/kernels/mhc/tilelang_kernels.py @@ -305,8 +305,12 @@ def mhc_pre_big_fuse_with_norm_tilelang( use_pre_mix_in: bool = False, save_pre_mix: bool = False, rms_numel: int = 0, + split_mode: str = "fused", write_aux: bool = False, ): + # Split modes require shifted mHC: input collapse uses a carried pre-mix. + assert split_mode in ("fused", "stats", "input") + assert split_mode == "fused" or save_pre_mix num_tokens = T.dynamic("num_tokens") hc_mult3 = hc_mult * (2 + hc_mult) if gemm_last_dim < 0: @@ -330,26 +334,26 @@ def mhc_pre_big_fuse_with_norm_tilelang( aux_out: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type] with T.Kernel(num_tokens, threads=96) as i: - rms = T.alloc_fragment(1, T.float32) - mixes = T.alloc_fragment(hc_mult3, T.float32) - T.clear(mixes) - rms[0] = 0 - if ENABLE_PDL: T.pdl_sync() - - for i_split in T.serial(n_splits): - rms[0] += gemm_out_sqrsum[i_split, i] - rms[0] = T.rsqrt(rms[0] / rms_numel + rms_eps) - for j in T.Parallel(hc_mult3): - mixes[j] = 0 - for i_split in T.serial(n_splits): - mixes[j] += gemm_out_mul[i_split, i, j] - mixes[j] *= rms[0] mixes_shared = T.alloc_shared(hc_mult3, T.float32) - T.copy(mixes, mixes_shared) + if split_mode != "input": + rms = T.alloc_fragment(1, T.float32) + mixes = T.alloc_fragment(hc_mult3, T.float32) + T.clear(mixes) + rms[0] = 0 - if T.get_thread_binding() < 32: + for i_split in T.serial(n_splits): + rms[0] += gemm_out_sqrsum[i_split, i] + rms[0] = T.rsqrt(rms[0] / rms_numel + rms_eps) + for j in T.Parallel(hc_mult3): + mixes[j] = 0 + for i_split in T.serial(n_splits): + mixes[j] += gemm_out_mul[i_split, i, j] + mixes[j] *= rms[0] + T.copy(mixes, mixes_shared) + + if split_mode != "input" and T.get_thread_binding() < 32: cm = T.alloc_fragment((hc_mult, hc_mult), T.float32) for j in T.Parallel(hc_mult): if save_pre_mix: @@ -395,7 +399,7 @@ def mhc_pre_big_fuse_with_norm_tilelang( for j, k in T.Parallel(hc_mult, hc_mult): comb_mix[i, j * hc_mult + k] = cm[j, k] - else: + elif split_mode != "stats" and T.get_thread_binding() >= 32: pre_mix_shared = T.alloc_shared(hc_mult, T.float32) for j in T.Parallel(hc_mult): if use_pre_mix_in: diff --git a/vllm/model_executor/kernels/mhc/warmup.py b/vllm/model_executor/kernels/mhc/warmup.py index 4e6a19781c..deb1cf429a 100644 --- a/vllm/model_executor/kernels/mhc/warmup.py +++ b/vllm/model_executor/kernels/mhc/warmup.py @@ -36,6 +36,7 @@ class MHCPreNormKernel(VllmJitKernel["MHCPreNormKernel.CompileKey"]): use_pre_mix_in: bool rms_numel: int save_pre_mix: bool = True + split_mode: str = "fused" write_aux: bool = False kernel: Any = staticmethod(mhc_pre_big_fuse_with_norm_tilelang) diff --git a/vllm/models/deepseek_v41/nvidia/model.py b/vllm/models/deepseek_v41/nvidia/model.py index 6d1faace65..b5caceab6f 100644 --- a/vllm/models/deepseek_v41/nvidia/model.py +++ b/vllm/models/deepseek_v41/nvidia/model.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import typing from collections.abc import Callable, Iterable +from functools import partial from itertools import islice import regex as re @@ -17,7 +18,11 @@ from vllm.distributed import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, ) -from vllm.forward_context import get_forward_context, is_forward_context_available +from vllm.forward_context import ( + get_forward_context, + in_piecewise_cudagraph, + is_forward_context_available, +) from vllm.logger import init_logger from vllm.model_executor.kernels.mhc.tilelang import ( mhc_post_tilelang, @@ -84,6 +89,11 @@ from ..common.engram import EngramLayout, NgramHashState from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask from .engram import Engram, gather_engram_hashes from .ops.mega_mhc import mhc_shifted_post_pre +from .ops.mhc import ( + MHC_OVERLAP_MAX_TOKENS, + mhc_pre_delayed_overlap, + supports_mhc_overlap, +) if typing.TYPE_CHECKING: from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata @@ -188,11 +198,13 @@ class DeepseekV4DecoderLayer(nn.Module): aux_stream_list: list[torch.cuda.Stream] | None = None, candidate_block_buffer: torch.Tensor | None = None, engram_layout: EngramLayout | None = None, + mhc_stream: torch.cuda.Stream | None = None, ): super().__init__() config = vllm_config.model_config.hf_config self.hidden_size = config.hidden_size + self.mhc_stream = mhc_stream self.use_sequence_parallel = _use_sequence_parallel(vllm_config) self.engram: Engram | None = None @@ -269,19 +281,30 @@ class DeepseekV4DecoderLayer(nn.Module): variants.append( {**hc_stream, "use_pre_mix_in": True, "write_aux": True} ) + # Shifted mHC splits the epilogue in two: the input collapse on the + # caller stream and coefficient generation on a side stream. + split_modes = ( + ("fused", "stats", "input") if mhc_stream is not None else ("fused",) + ) for variant in variants: - MHC_PRE_NORM_KERNEL.register_warmup( - max_tokens=max_tokens, - hidden_size=self.hidden_size, - rms_eps=self.rms_norm_eps, - hc_pre_eps=self.hc_eps, - hc_sinkhorn_eps=self.hc_eps, - hc_post_mult_value=self.hc_post_alpha, - sinkhorn_repeat=self.hc_sinkhorn_iters, - norm_eps=self.rms_norm_eps, - hc_mult=self.hc_mult, - **variant, - ) + for split_mode in split_modes: + MHC_PRE_NORM_KERNEL.register_warmup( + split_mode=split_mode, + max_tokens=( + max_tokens + if split_mode == "fused" + else min(max_tokens, MHC_OVERLAP_MAX_TOKENS) + ), + hidden_size=self.hidden_size, + rms_eps=self.rms_norm_eps, + hc_pre_eps=self.hc_eps, + hc_sinkhorn_eps=self.hc_eps, + hc_post_mult_value=self.hc_post_alpha, + sinkhorn_repeat=self.hc_sinkhorn_iters, + norm_eps=self.rms_norm_eps, + hc_mult=self.hc_mult, + **variant, + ) mix_hc = (2 + self.hc_mult) * self.hc_mult hc_dim = self.hc_mult * self.hidden_size self.hc_attn_fn = nn.Parameter( @@ -351,6 +374,18 @@ class DeepseekV4DecoderLayer(nn.Module): torch.Tensor | None, ]: previous_aux: torch.Tensor | None = None + mhc_stream = self.mhc_stream + if mhc_stream is not None and ( + in_piecewise_cudagraph() + or not 0 < positions.shape[0] <= MHC_OVERLAP_MAX_TOKENS + ): + # A side stream cannot remain unjoined across breakable graph segments. + mhc_stream = None + mhc_pre = ( + partial(mhc_pre_delayed_overlap, stream=mhc_stream) + if mhc_stream is not None + else mhc_pre_delayed_tilelang + ) # The reference collapses each sublayer's input with the *previous* # sublayer's pre-mix: attention uses the pre-mix carried in (identity # for the first layer), the FFN uses this layer's attention pre-mix. @@ -360,7 +395,7 @@ class DeepseekV4DecoderLayer(nn.Module): # copies and the identity pre-mix selects copy 0. assert self.hc_attn_fn_broadcast is not None residual = x.unsqueeze(1).expand(-1, self.hc_mult, -1).contiguous() - post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang( + post_mix, res_mix, x, attn_pre = mhc_pre( residual, self.hc_attn_fn_broadcast, self.hc_attn_scale, @@ -376,7 +411,7 @@ class DeepseekV4DecoderLayer(nn.Module): ) else: residual = x - post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang( + post_mix, res_mix, x, attn_pre = mhc_pre( residual, self.hc_attn_fn, self.hc_attn_scale, @@ -403,7 +438,7 @@ class DeepseekV4DecoderLayer(nn.Module): engram_hashes[:, self.engram.layer_hash_index], engram_mask, ) - post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang( + post_mix, res_mix, x, attn_pre = mhc_pre( residual, self.hc_attn_fn, self.hc_attn_scale, @@ -437,6 +472,7 @@ class DeepseekV4DecoderLayer(nn.Module): norm_weight=self.attn_norm.weight, norm_eps=self.attn_norm.variance_epsilon, capture_aux=capture_previous_aux, + stream=mhc_stream, ) if capture_previous_aux: previous_aux = aux @@ -448,6 +484,8 @@ class DeepseekV4DecoderLayer(nn.Module): if self.use_sequence_parallel: x = sp_reduce_scatter(x) + if mhc_stream is not None: + torch.cuda.current_stream().wait_stream(mhc_stream) residual, post_mix, res_mix, x, ffn_pre, _ = mhc_shifted_post_pre( x, residual, @@ -464,8 +502,11 @@ class DeepseekV4DecoderLayer(nn.Module): pre_mix=attn_pre, norm_weight=self.ffn_norm.weight, norm_eps=self.ffn_norm.variance_epsilon, + stream=mhc_stream, ) x = self.ffn(x, input_ids, mega_gate_metadata) + if mhc_stream is not None: + torch.cuda.current_stream().wait_stream(mhc_stream) return x, residual, post_mix, res_mix, ffn_pre, previous_aux @@ -497,6 +538,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): # (compressor kv_score, indexer.weights_proj). fused_wqa_wkv stays on # the default stream. aux_stream_list = [torch.cuda.Stream() for _ in range(3)] + # Keep mHC independent of the streams used inside attention. + mhc_stream = torch.cuda.Stream() if supports_mhc_overlap(vllm_config) else None # Reserved topk indices buffer for all Indexer layers to reuse. self.topk_indices_buffer = torch.empty( @@ -541,6 +584,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): aux_stream_list=aux_stream_list, candidate_block_buffer=self.candidate_block_buffer, engram_layout=self.engram_layout, + mhc_stream=mhc_stream, ), prefix=f"{prefix}.layers", ) diff --git a/vllm/models/deepseek_v41/nvidia/ops/mega_mhc.py b/vllm/models/deepseek_v41/nvidia/ops/mega_mhc.py index f2c7490261..caae3d8d9b 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/mega_mhc.py +++ b/vllm/models/deepseek_v41/nvidia/ops/mega_mhc.py @@ -7,6 +7,7 @@ import torch from vllm.model_executor.kernels.mhc.tilelang import ( mhc_fused_post_pre_delayed_tilelang, + mhc_post_tilelang, ) from vllm.platforms import current_platform from vllm.utils.deep_gemm import ( @@ -106,10 +107,37 @@ def mhc_shifted_post_pre( norm_weight: torch.Tensor | None = None, norm_eps: float = 1e-6, capture_aux: bool = False, + *, + stream: torch.cuda.Stream | None = None, ) -> tuple[ torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor ]: - """Use Mega-mHC for carried mixing, retaining TileLang for aux capture.""" + """Dispatch shifted post/pre to overlap, Mega-mHC, or fused TileLang. + + When stream is supplied, join it before consuming the returned coefficients. + """ + if stream is not None: + from .mhc import mhc_pre_delayed_overlap + + residual = mhc_post_tilelang(x, residual, post_layer_mix, comb_res_mix) + aux = residual.mean(dim=1) if capture_aux else x.new_empty(0, x.shape[1]) + pre_outputs = mhc_pre_delayed_overlap( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + pre_mix=pre_mix, + norm_weight=norm_weight, + norm_eps=norm_eps, + stream=stream, + ) + return residual, *pre_outputs, aux + if ( pre_mix is not None and norm_weight is not None diff --git a/vllm/models/deepseek_v41/nvidia/ops/mhc.py b/vllm/models/deepseek_v41/nvidia/ops/mhc.py new file mode 100644 index 0000000000..29c9c3b956 --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/ops/mhc.py @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Split shifted mHC pre so coefficient generation can overlap the sublayer.""" + +from functools import partial + +import torch + +from vllm.config import VllmConfig +from vllm.platforms import current_platform +from vllm.utils.deep_gemm import is_deep_gemm_supported + +# GB200 TP4/FlashInfer improves through 16 tokens; larger screens tie or regress. +MHC_OVERLAP_MAX_TOKENS = 16 + + +def supports_mhc_overlap(vllm_config: VllmConfig) -> bool: + """Check kernel requirements and safety of sharing the coefficient stream.""" + config = vllm_config.model_config.hf_config + # DeepGEMM's prenorm kernel requires K % 64 == 0, N % 8 == 0, and N <= 32. + mix_size = config.hc_mult * (config.hc_mult + 2) + return ( + current_platform.is_device_capability_family(100) + and is_deep_gemm_supported() + and config.hidden_size % 64 == 0 + and 0 < mix_size <= 32 + and mix_size % 8 == 0 + and not vllm_config.parallel_config.use_ubatching + ) + + +def mhc_pre_delayed_overlap( + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None = None, + x: torch.Tensor | None = None, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 1e-6, + *, + stream: torch.cuda.Stream, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Prepare the input on the caller stream and coefficients on another stream. + + Returns post mix, residual mix, normalized input, and next pre-mix. Only the + input is ready on the caller stream; join stream before using coefficients. + The caller must retain the weights and coefficient outputs until the join. + """ + from vllm.model_executor.kernels.mhc.warmup import ( + MHC_PRE_NORM_KERNEL, + compute_mhc_pre_num_splits, + ) + from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm + + n, hc, hidden = residual.shape + assert residual.is_contiguous() and residual.dtype == torch.bfloat16 + assert norm_weight is not None + if x is None: + x = residual.view(n, hc * hidden) + assert x.is_contiguous() + post = torch.empty((n, hc), device=residual.device, dtype=torch.float32) + comb = torch.empty((n, hc * hc), device=residual.device, dtype=torch.float32) + next_pre = torch.empty_like(post) + layer_input = torch.empty((n, hidden), device=residual.device, dtype=torch.bfloat16) + outputs = post.unsqueeze(-1), comb.view(n, hc, hc), layer_input, next_pre + if n == 0: + return outputs + splits = compute_mhc_pre_num_splits(x.shape[1], n) + mix = torch.empty( + (splits, n, hc * (hc + 2)), device=residual.device, dtype=torch.float32 + ) + sqr = torch.empty((splits, n), device=residual.device, dtype=torch.float32) + epilogue = partial( + MHC_PRE_NORM_KERNEL, + mix, + sqr, + hc_scale, + hc_base, + residual, + post, + comb, + layer_input, + norm_weight, + pre_mix if pre_mix is not None else post, + next_pre, + layer_input, # Unused aux output in split modes. + hidden_size=hidden, + rms_eps=rms_eps, + hc_pre_eps=hc_pre_eps, + hc_sinkhorn_eps=hc_sinkhorn_eps, + hc_post_mult_value=hc_post_mult_value, + sinkhorn_repeat=sinkhorn_repeat, + norm_eps=norm_eps, + hc_mult=hc, + use_pre_mix_in=pre_mix is not None, + save_pre_mix=True, + rms_numel=x.shape[1], + ) + main = torch.cuda.current_stream() + # Prioritize input readiness for small batches before releasing statistics. + if n <= 8: + epilogue(split_mode="input") + stream.wait_stream(main) + with torch.cuda.stream(stream): + tf32_hc_prenorm_gemm(x, fn, mix, sqr, splits) + epilogue(split_mode="stats") + for tensor in (x, mix, sqr): + tensor.record_stream(stream) + if n > 8: + epilogue(split_mode="input") + return outputs