[Perf][DSV4.1] Fuse TP all-reduce with mHC input preparation (#57643)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -50,6 +50,40 @@ from vllm.utils.torch_utils import set_random_seed
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp,ep,hidden,hc,multicast,expected",
|
||||
[
|
||||
(4, False, 5120, 4, 1, True),
|
||||
(2, False, 5120, 4, 1, False),
|
||||
(4, True, 5120, 4, 1, False),
|
||||
(4, False, 4096, 4, 1, False),
|
||||
(4, False, 5120, 2, 1, False),
|
||||
(4, False, 5120, 4, 0, False),
|
||||
],
|
||||
)
|
||||
def test_deepseek_v41_all_reduce_fusion_requires_kernel_support(
|
||||
monkeypatch, tp, ep, hidden, hc, multicast, expected
|
||||
):
|
||||
"""Eligibility depends on kernel inputs, not serving or attention settings."""
|
||||
from vllm.models.deepseek_v41.nvidia.ops import mhc
|
||||
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
tensor_parallel_size=tp, enable_expert_parallel=ep
|
||||
),
|
||||
model_config=SimpleNamespace(
|
||||
hf_config=SimpleNamespace(hidden_size=hidden, hc_mult=hc)
|
||||
),
|
||||
)
|
||||
group = SimpleNamespace(
|
||||
device_communicator=SimpleNamespace(
|
||||
ca_comm=SimpleNamespace(mnnvl_lamport_ag_multicast_ptr=multicast)
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(mhc, "get_tp_group", lambda: group)
|
||||
assert mhc.supports_mhc_all_reduce(config) is expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens,hc_mult,hidden_size",
|
||||
@@ -523,6 +557,7 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
decoder.rms_norm_eps = 1e-20
|
||||
decoder.hc_post_alpha = 2.0
|
||||
decoder.use_sequence_parallel = False
|
||||
decoder.fuse_mhc_all_reduce = False
|
||||
decoder.mhc_stream = None
|
||||
if mhc_mode != "disabled":
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
@@ -597,7 +632,15 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
return post, res, decoder.attn_norm(collapsed), pre
|
||||
|
||||
def fused_reference(
|
||||
x, residual, post_mix, res_mix, *args, capture_aux=False, stream=None, **kw
|
||||
x,
|
||||
residual,
|
||||
post_mix,
|
||||
res_mix,
|
||||
*args,
|
||||
capture_aux=False,
|
||||
stream=None,
|
||||
reduce_results=False,
|
||||
**kw,
|
||||
):
|
||||
residual = mhc_post_torch(x, residual, post_mix, res_mix)
|
||||
aux = residual.mean(dim=1) if capture_aux else residual.new_empty(0)
|
||||
@@ -638,6 +681,7 @@ def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_conf
|
||||
decoder.rms_norm_eps = 1e-6
|
||||
decoder.hc_post_alpha = 2.0
|
||||
decoder.use_sequence_parallel = False
|
||||
decoder.fuse_mhc_all_reduce = False
|
||||
decoder.mhc_stream = None
|
||||
decoder.engram = None
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
|
||||
Reference in New Issue
Block a user