[Bugfix][DSV4.1] Restrict mHC overlap to full CUDA graphs (#57874)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
97dc6b19d2
commit
67513c8b67
@@ -539,7 +539,7 @@ def test_mhc_fused_post_pre_delayed_custom_op_supports_compile(carried, capture_
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize("entry", ["broadcast", "pipeline", "residual", "engram"])
|
||||
@pytest.mark.parametrize(
|
||||
"mhc_mode", ["disabled", "overlap", "piecewise", "large_batch"]
|
||||
"mhc_mode", ["disabled", "eager", "overlap", "piecewise", "large_batch"]
|
||||
)
|
||||
def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
entry, mhc_mode, monkeypatch, default_vllm_config
|
||||
@@ -568,7 +568,7 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
):
|
||||
pytest.skip("SM100 DeepGEMM required for overlap")
|
||||
decoder.mhc_stream = torch.cuda.Stream()
|
||||
if mhc_mode in ("piecewise", "large_batch"):
|
||||
if mhc_mode in ("eager", "piecewise", "large_batch"):
|
||||
|
||||
def unexpected_overlap(*args, **kwargs):
|
||||
pytest.fail("unsupported execution must retain native mHC")
|
||||
@@ -624,8 +624,27 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
from vllm.forward_context import set_forward_context
|
||||
|
||||
mode = CUDAGraphMode.PIECEWISE if mhc_mode == "piecewise" else CUDAGraphMode.NONE
|
||||
overlap_calls = []
|
||||
if mhc_mode == "overlap":
|
||||
from vllm.models.deepseek_v41.nvidia.ops.mhc import mhc_pre_delayed_overlap
|
||||
|
||||
def checked_overlap(*args, **kwargs):
|
||||
overlap_calls.append(torch.cuda.is_current_stream_capturing())
|
||||
return mhc_pre_delayed_overlap(*args, **kwargs)
|
||||
|
||||
for module in ("model", "ops.mhc"):
|
||||
monkeypatch.setattr(
|
||||
f"vllm.models.deepseek_v41.nvidia.{module}.mhc_pre_delayed_overlap",
|
||||
checked_overlap,
|
||||
)
|
||||
with set_forward_context(None, default_vllm_config, cudagraph_runtime_mode=mode):
|
||||
actual = decoder(x, positions, None, **kwargs)
|
||||
if mhc_mode == "overlap":
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = decoder(x, positions, None, **kwargs)
|
||||
graph.replay()
|
||||
assert overlap_calls and all(overlap_calls)
|
||||
|
||||
def reference(*args, norm_weight, norm_eps, **kwargs):
|
||||
post, res, collapsed, pre = mhc_pre_delayed_torch(*args, **kwargs)
|
||||
|
||||
@@ -385,8 +385,9 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
if mhc_stream is not None and (
|
||||
in_piecewise_cudagraph()
|
||||
or not 0 < positions.shape[0] <= MHC_OVERLAP_MAX_TOKENS
|
||||
or not torch.cuda.is_current_stream_capturing()
|
||||
):
|
||||
# A side stream cannot remain unjoined across breakable graph segments.
|
||||
# Use overlap only in FULL graphs; eager warmup initializes Mega mHC.
|
||||
mhc_stream = None
|
||||
mhc_pre = (
|
||||
partial(mhc_pre_delayed_overlap, stream=mhc_stream)
|
||||
|
||||
@@ -28,6 +28,22 @@ def is_mega_mhc_supported(hidden_size: int, hc_mult: int) -> bool:
|
||||
return deep_gemm is not None and callable(getattr(deep_gemm, "mega_mhc", None))
|
||||
|
||||
|
||||
def can_use_mega_mhc(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
pre_mix: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor | None,
|
||||
capture_aux: bool,
|
||||
) -> bool:
|
||||
return (
|
||||
pre_mix is not None
|
||||
and norm_weight is not None
|
||||
and not capture_aux
|
||||
and x.shape[0] <= 1 << 20
|
||||
and is_mega_mhc_supported(x.shape[1], residual.shape[1])
|
||||
)
|
||||
|
||||
|
||||
def mhc_shifted_post_pre_deep_gemm(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
|
||||
@@ -16,7 +16,10 @@ from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
from .mega_mhc import is_mega_mhc_supported, mhc_shifted_post_pre_deep_gemm
|
||||
from .mega_mhc import (
|
||||
can_use_mega_mhc,
|
||||
mhc_shifted_post_pre_deep_gemm,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
|
||||
@@ -223,13 +226,8 @@ def mhc_shifted_post_pre(
|
||||
)
|
||||
return residual, *pre_outputs, aux
|
||||
|
||||
if (
|
||||
pre_mix is not None
|
||||
and norm_weight is not None
|
||||
and not capture_aux
|
||||
and x.shape[0] <= 1 << 20
|
||||
and is_mega_mhc_supported(x.shape[1], residual.shape[1])
|
||||
):
|
||||
if can_use_mega_mhc(x, residual, pre_mix, norm_weight, capture_aux):
|
||||
assert pre_mix is not None and norm_weight is not None
|
||||
outputs = mhc_shifted_post_pre_deep_gemm(
|
||||
x,
|
||||
residual,
|
||||
|
||||
Reference in New Issue
Block a user