[Bugfix][DSV4.1] Restrict mHC overlap to full CUDA graphs (#57874)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Woosuk Kwon
2026-09-21 01:16:59 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 97dc6b19d2
commit 67513c8b67
4 changed files with 45 additions and 11 deletions
+2 -1
View File
@@ -385,8 +385,9 @@ class DeepseekV4DecoderLayer(nn.Module):
if mhc_stream is not None and (
in_piecewise_cudagraph()
or not 0 < positions.shape[0] <= MHC_OVERLAP_MAX_TOKENS
or not torch.cuda.is_current_stream_capturing()
):
# A side stream cannot remain unjoined across breakable graph segments.
# Use overlap only in FULL graphs; eager warmup initializes Mega mHC.
mhc_stream = None
mhc_pre = (
partial(mhc_pre_delayed_overlap, stream=mhc_stream)
@@ -28,6 +28,22 @@ def is_mega_mhc_supported(hidden_size: int, hc_mult: int) -> bool:
return deep_gemm is not None and callable(getattr(deep_gemm, "mega_mhc", None))
def can_use_mega_mhc(
x: torch.Tensor,
residual: torch.Tensor,
pre_mix: torch.Tensor | None,
norm_weight: torch.Tensor | None,
capture_aux: bool,
) -> bool:
return (
pre_mix is not None
and norm_weight is not None
and not capture_aux
and x.shape[0] <= 1 << 20
and is_mega_mhc_supported(x.shape[1], residual.shape[1])
)
def mhc_shifted_post_pre_deep_gemm(
x: torch.Tensor,
residual: torch.Tensor,
+6 -8
View File
@@ -16,7 +16,10 @@ from vllm.model_executor.kernels.mhc.tilelang import (
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
from .mega_mhc import is_mega_mhc_supported, mhc_shifted_post_pre_deep_gemm
from .mega_mhc import (
can_use_mega_mhc,
mhc_shifted_post_pre_deep_gemm,
)
if TYPE_CHECKING:
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
@@ -223,13 +226,8 @@ def mhc_shifted_post_pre(
)
return residual, *pre_outputs, aux
if (
pre_mix is not None
and norm_weight is not None
and not capture_aux
and x.shape[0] <= 1 << 20
and is_mega_mhc_supported(x.shape[1], residual.shape[1])
):
if can_use_mega_mhc(x, residual, pre_mix, norm_weight, capture_aux):
assert pre_mix is not None and norm_weight is not None
outputs = mhc_shifted_post_pre_deep_gemm(
x,
residual,