[Perf][DSV4.1] Fuse TP all-reduce with mHC input preparation (#57643)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Woosuk Kwon
2026-09-20 20:28:06 -07:00
committed by GitHub
co-authored by Codex
parent eb87980585
commit 9a70c233cd
9 changed files with 549 additions and 105 deletions
+111
View File
@@ -11,6 +11,7 @@ import torch.distributed as dist
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
from vllm.distributed.device_communicators import custom_all_reduce as car
from vllm.distributed.parallel_state import get_tp_group, graph_capture
from vllm.platforms import current_platform
from ..utils import (
ensure_model_parallel_initialized,
@@ -24,6 +25,116 @@ for i, v in enumerate(test_sizes):
test_sizes[i] -= v % 8
@ray.remote(num_gpus=1, max_calls=1)
def _all_reduce_mhc(monkeypatch, tp_size, pp_size, rank, distributed_init_port):
from vllm.model_executor.kernels.mhc.tilelang import (
mhc_post_tilelang,
mhc_pre_delayed_tilelang,
)
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
comm = get_tp_group().device_communicator.ca_comm
assert comm is not None and comm.mnnvl_lamport_ag_multicast_ptr
fn = torch.zeros(24, 20480, device=device)
scale = torch.ones(3, device=device)
base = torch.zeros(24, device=device)
# Changing shapes and interleaving all-gather exercise the shared
# Lamport stages, including cleanup of a larger previous payload.
def run(n):
torch.manual_seed(42 + rank)
x = torch.randn(n, 5120, device=device, dtype=torch.bfloat16)
# Packed +0/-0 pairs collide with the Lamport sentinel.
x[:, :16] = 0
x[:, 9:16:2] = -0.0
torch.manual_seed(123)
residual = torch.randn(n, 4, 5120, device=device, dtype=torch.bfloat16)
post = torch.rand(n, 4, device=device)
comb = torch.randn(n, 4, 4, device=device) * 0.1
pre = torch.rand(n, 4, device=device)
weight = torch.randn(5120, device=device, dtype=torch.bfloat16)
output = torch.empty_like(residual)
normalized = torch.empty_like(x)
def fused():
torch.ops._C_custom_ar.all_reduce_mhc(
x,
residual,
post,
comb,
pre,
weight,
output,
normalized,
comm.mnnvl_lamport_ag_local_ptr,
comm.mnnvl_lamport_ag_multicast_ptr,
comm.mnnvl_lamport_epochs[0],
rank,
comm.mnnvl_buffer_size,
1e-6,
)
def check():
gathered = comm.custom_all_gather(x)
assert gathered is not None
peers = gathered.view(tp_size, n, 5120).float()
reduced = peers[0].clone()
for peer in peers[1:]:
reduced.add_(peer)
expected = mhc_post_tilelang(
reduced.bfloat16(), residual, post.unsqueeze(-1), comb
)
expected_norm = mhc_pre_delayed_tilelang(
expected,
fn,
scale,
base,
1e-6,
1e-6,
1e-6,
2.0,
20,
pre_mix=pre,
norm_weight=weight,
)[2]
torch.testing.assert_close(output, expected, rtol=0.008, atol=1e-6)
torch.testing.assert_close(
normalized, expected_norm, rtol=0.008, atol=0.008
)
fused()
check()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
for _ in range(6):
fused()
for _ in range(20):
graph.replay()
check()
x.mul_(0.5)
residual.mul_(2)
graph.replay()
check()
# Cover fixed Q6 and shrinking/growing adaptive verification batches.
for n in (1, 6, 12, 8, 16, 3, 5, 2, 4, 1):
run(n)
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100), reason="Requires SM100"
)
def test_all_reduce_mhc_preserves_bf16_boundaries_and_graph_replay(monkeypatch):
if torch.accelerator.device_count() < 4:
pytest.skip("Requires four GPUs with NVLink multicast")
multi_process_parallel(monkeypatch, 4, 1, _all_reduce_mhc)
@pytest.mark.parametrize(
("dtype", "expected"),
[
@@ -512,6 +512,7 @@ def _check_dummy_hash_model_forward(
model = SimpleNamespace(
use_mega_moe=False,
use_sequence_parallel=False,
fuse_mhc_all_reduce=False,
engram_hash=state,
engram_swa_prefix="swa",
engram_dp_shared_memory=dp_shared_memory,
+45 -1
View File
@@ -50,6 +50,40 @@ from vllm.utils.torch_utils import set_random_seed
DEVICE = current_platform.device_type
@pytest.mark.parametrize(
"tp,ep,hidden,hc,multicast,expected",
[
(4, False, 5120, 4, 1, True),
(2, False, 5120, 4, 1, False),
(4, True, 5120, 4, 1, False),
(4, False, 4096, 4, 1, False),
(4, False, 5120, 2, 1, False),
(4, False, 5120, 4, 0, False),
],
)
def test_deepseek_v41_all_reduce_fusion_requires_kernel_support(
monkeypatch, tp, ep, hidden, hc, multicast, expected
):
"""Eligibility depends on kernel inputs, not serving or attention settings."""
from vllm.models.deepseek_v41.nvidia.ops import mhc
config = SimpleNamespace(
parallel_config=SimpleNamespace(
tensor_parallel_size=tp, enable_expert_parallel=ep
),
model_config=SimpleNamespace(
hf_config=SimpleNamespace(hidden_size=hidden, hc_mult=hc)
),
)
group = SimpleNamespace(
device_communicator=SimpleNamespace(
ca_comm=SimpleNamespace(mnnvl_lamport_ag_multicast_ptr=multicast)
)
)
monkeypatch.setattr(mhc, "get_tp_group", lambda: group)
assert mhc.supports_mhc_all_reduce(config) is expected
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
@pytest.mark.parametrize(
"num_tokens,hc_mult,hidden_size",
@@ -523,6 +557,7 @@ def test_deepseek_v41_decoder_mixes_match_torch(
decoder.rms_norm_eps = 1e-20
decoder.hc_post_alpha = 2.0
decoder.use_sequence_parallel = False
decoder.fuse_mhc_all_reduce = False
decoder.mhc_stream = None
if mhc_mode != "disabled":
from vllm.utils.deep_gemm import is_deep_gemm_supported
@@ -597,7 +632,15 @@ def test_deepseek_v41_decoder_mixes_match_torch(
return post, res, decoder.attn_norm(collapsed), pre
def fused_reference(
x, residual, post_mix, res_mix, *args, capture_aux=False, stream=None, **kw
x,
residual,
post_mix,
res_mix,
*args,
capture_aux=False,
stream=None,
reduce_results=False,
**kw,
):
residual = mhc_post_torch(x, residual, post_mix, res_mix)
aux = residual.mean(dim=1) if capture_aux else residual.new_empty(0)
@@ -638,6 +681,7 @@ def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_conf
decoder.rms_norm_eps = 1e-6
decoder.hc_post_alpha = 2.0
decoder.use_sequence_parallel = False
decoder.fuse_mhc_all_reduce = False
decoder.mhc_stream = None
decoder.engram = None
from vllm.model_executor.layers.layernorm import RMSNorm