From 67513c8b67e2439934ce7c0ba5a18c2e859f5c53 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Mon, 21 Sep 2026 01:16:59 -0700 Subject: [PATCH] [Bugfix][DSV4.1] Restrict mHC overlap to full CUDA graphs (#57874) Signed-off-by: Woosuk Kwon Co-authored-by: OpenAI Codex --- tests/kernels/test_mhc_kernels.py | 23 +++++++++++++++++-- vllm/models/deepseek_v41/nvidia/model.py | 3 ++- .../deepseek_v41/nvidia/ops/mega_mhc.py | 16 +++++++++++++ vllm/models/deepseek_v41/nvidia/ops/mhc.py | 14 +++++------ 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/tests/kernels/test_mhc_kernels.py b/tests/kernels/test_mhc_kernels.py index 63648ebf2d..4735fb96d1 100644 --- a/tests/kernels/test_mhc_kernels.py +++ b/tests/kernels/test_mhc_kernels.py @@ -539,7 +539,7 @@ def test_mhc_fused_post_pre_delayed_custom_op_supports_compile(carried, capture_ @pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required") @pytest.mark.parametrize("entry", ["broadcast", "pipeline", "residual", "engram"]) @pytest.mark.parametrize( - "mhc_mode", ["disabled", "overlap", "piecewise", "large_batch"] + "mhc_mode", ["disabled", "eager", "overlap", "piecewise", "large_batch"] ) def test_deepseek_v41_decoder_mixes_match_torch( entry, mhc_mode, monkeypatch, default_vllm_config @@ -568,7 +568,7 @@ def test_deepseek_v41_decoder_mixes_match_torch( ): pytest.skip("SM100 DeepGEMM required for overlap") decoder.mhc_stream = torch.cuda.Stream() - if mhc_mode in ("piecewise", "large_batch"): + if mhc_mode in ("eager", "piecewise", "large_batch"): def unexpected_overlap(*args, **kwargs): pytest.fail("unsupported execution must retain native mHC") @@ -624,8 +624,27 @@ def test_deepseek_v41_decoder_mixes_match_torch( from vllm.forward_context import set_forward_context mode = CUDAGraphMode.PIECEWISE if mhc_mode == "piecewise" else CUDAGraphMode.NONE + overlap_calls = [] + if mhc_mode == "overlap": + from vllm.models.deepseek_v41.nvidia.ops.mhc import mhc_pre_delayed_overlap + + def checked_overlap(*args, **kwargs): + overlap_calls.append(torch.cuda.is_current_stream_capturing()) + return mhc_pre_delayed_overlap(*args, **kwargs) + + for module in ("model", "ops.mhc"): + monkeypatch.setattr( + f"vllm.models.deepseek_v41.nvidia.{module}.mhc_pre_delayed_overlap", + checked_overlap, + ) with set_forward_context(None, default_vllm_config, cudagraph_runtime_mode=mode): actual = decoder(x, positions, None, **kwargs) + if mhc_mode == "overlap": + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = decoder(x, positions, None, **kwargs) + graph.replay() + assert overlap_calls and all(overlap_calls) def reference(*args, norm_weight, norm_eps, **kwargs): post, res, collapsed, pre = mhc_pre_delayed_torch(*args, **kwargs) diff --git a/vllm/models/deepseek_v41/nvidia/model.py b/vllm/models/deepseek_v41/nvidia/model.py index d7c9ddc128..b91ad1d7ba 100644 --- a/vllm/models/deepseek_v41/nvidia/model.py +++ b/vllm/models/deepseek_v41/nvidia/model.py @@ -385,8 +385,9 @@ class DeepseekV4DecoderLayer(nn.Module): if mhc_stream is not None and ( in_piecewise_cudagraph() or not 0 < positions.shape[0] <= MHC_OVERLAP_MAX_TOKENS + or not torch.cuda.is_current_stream_capturing() ): - # A side stream cannot remain unjoined across breakable graph segments. + # Use overlap only in FULL graphs; eager warmup initializes Mega mHC. mhc_stream = None mhc_pre = ( partial(mhc_pre_delayed_overlap, stream=mhc_stream) diff --git a/vllm/models/deepseek_v41/nvidia/ops/mega_mhc.py b/vllm/models/deepseek_v41/nvidia/ops/mega_mhc.py index ef56b5a823..bdb5b95bbe 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/mega_mhc.py +++ b/vllm/models/deepseek_v41/nvidia/ops/mega_mhc.py @@ -28,6 +28,22 @@ def is_mega_mhc_supported(hidden_size: int, hc_mult: int) -> bool: return deep_gemm is not None and callable(getattr(deep_gemm, "mega_mhc", None)) +def can_use_mega_mhc( + x: torch.Tensor, + residual: torch.Tensor, + pre_mix: torch.Tensor | None, + norm_weight: torch.Tensor | None, + capture_aux: bool, +) -> bool: + return ( + pre_mix is not None + and norm_weight is not None + and not capture_aux + and x.shape[0] <= 1 << 20 + and is_mega_mhc_supported(x.shape[1], residual.shape[1]) + ) + + def mhc_shifted_post_pre_deep_gemm( x: torch.Tensor, residual: torch.Tensor, diff --git a/vllm/models/deepseek_v41/nvidia/ops/mhc.py b/vllm/models/deepseek_v41/nvidia/ops/mhc.py index 084a102436..88288ead4d 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/mhc.py +++ b/vllm/models/deepseek_v41/nvidia/ops/mhc.py @@ -16,7 +16,10 @@ from vllm.model_executor.kernels.mhc.tilelang import ( from vllm.platforms import current_platform from vllm.utils.deep_gemm import is_deep_gemm_supported -from .mega_mhc import is_mega_mhc_supported, mhc_shifted_post_pre_deep_gemm +from .mega_mhc import ( + can_use_mega_mhc, + mhc_shifted_post_pre_deep_gemm, +) if TYPE_CHECKING: from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator @@ -223,13 +226,8 @@ def mhc_shifted_post_pre( ) return residual, *pre_outputs, aux - if ( - pre_mix is not None - and norm_weight is not None - and not capture_aux - and x.shape[0] <= 1 << 20 - and is_mega_mhc_supported(x.shape[1], residual.shape[1]) - ): + if can_use_mega_mhc(x, residual, pre_mix, norm_weight, capture_aux): + assert pre_mix is not None and norm_weight is not None outputs = mhc_shifted_post_pre_deep_gemm( x, residual,