[Perf][DSV4.1] Fuse TP all-reduce with mHC input preparation (#57643)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Woosuk Kwon
2026-09-20 20:28:06 -07:00
committed by GitHub
co-authored by Codex
parent eb87980585
commit 9a70c233cd
9 changed files with 549 additions and 105 deletions
+45 -1
View File
@@ -50,6 +50,40 @@ from vllm.utils.torch_utils import set_random_seed
DEVICE = current_platform.device_type
@pytest.mark.parametrize(
"tp,ep,hidden,hc,multicast,expected",
[
(4, False, 5120, 4, 1, True),
(2, False, 5120, 4, 1, False),
(4, True, 5120, 4, 1, False),
(4, False, 4096, 4, 1, False),
(4, False, 5120, 2, 1, False),
(4, False, 5120, 4, 0, False),
],
)
def test_deepseek_v41_all_reduce_fusion_requires_kernel_support(
monkeypatch, tp, ep, hidden, hc, multicast, expected
):
"""Eligibility depends on kernel inputs, not serving or attention settings."""
from vllm.models.deepseek_v41.nvidia.ops import mhc
config = SimpleNamespace(
parallel_config=SimpleNamespace(
tensor_parallel_size=tp, enable_expert_parallel=ep
),
model_config=SimpleNamespace(
hf_config=SimpleNamespace(hidden_size=hidden, hc_mult=hc)
),
)
group = SimpleNamespace(
device_communicator=SimpleNamespace(
ca_comm=SimpleNamespace(mnnvl_lamport_ag_multicast_ptr=multicast)
)
)
monkeypatch.setattr(mhc, "get_tp_group", lambda: group)
assert mhc.supports_mhc_all_reduce(config) is expected
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
@pytest.mark.parametrize(
"num_tokens,hc_mult,hidden_size",
@@ -523,6 +557,7 @@ def test_deepseek_v41_decoder_mixes_match_torch(
decoder.rms_norm_eps = 1e-20
decoder.hc_post_alpha = 2.0
decoder.use_sequence_parallel = False
decoder.fuse_mhc_all_reduce = False
decoder.mhc_stream = None
if mhc_mode != "disabled":
from vllm.utils.deep_gemm import is_deep_gemm_supported
@@ -597,7 +632,15 @@ def test_deepseek_v41_decoder_mixes_match_torch(
return post, res, decoder.attn_norm(collapsed), pre
def fused_reference(
x, residual, post_mix, res_mix, *args, capture_aux=False, stream=None, **kw
x,
residual,
post_mix,
res_mix,
*args,
capture_aux=False,
stream=None,
reduce_results=False,
**kw,
):
residual = mhc_post_torch(x, residual, post_mix, res_mix)
aux = residual.mean(dim=1) if capture_aux else residual.new_empty(0)
@@ -638,6 +681,7 @@ def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_conf
decoder.rms_norm_eps = 1e-6
decoder.hc_post_alpha = 2.0
decoder.use_sequence_parallel = False
decoder.fuse_mhc_all_reduce = False
decoder.mhc_stream = None
decoder.engram = None
from vllm.model_executor.layers.layernorm import RMSNorm