[Kernel][DSV4.1] Fuse MoE finalize into the TP all-reduce + mHC boundary (#58586)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-27 03:08:36 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 379e9a1ea8
commit 77871126f9
8 changed files with 1804 additions and 306 deletions
+117 -70
View File
@@ -25,12 +25,43 @@ for i, v in enumerate(test_sizes):
test_sizes[i] -= v % 8
def _bf16_ulps(a: torch.Tensor, b: torch.Tensor) -> int:
"""Largest distance between two BF16 tensors in units in the last place."""
def ordered(t: torch.Tensor) -> torch.Tensor:
bits = t.view(torch.int16).int() & 0xFFFF
return torch.where(bits >= 0x8000, 0x8000 - bits, bits)
return int((ordered(a) - ordered(b)).abs().max())
def _unfused_all_reduce_mhc(peers, residual, post, comb, pre, weight, eps):
"""All-reduce, mHC post, collapse and RMSNorm in the fused kernel's FP32
order, rounding to BF16 where the unfused path returns BF16."""
reduced = peers[0].float()
for peer in peers[1:]:
reduced = reduced + peer.float()
reduced = reduced.bfloat16().float()
output = torch.empty_like(residual)
collapse = torch.zeros_like(reduced)
for target in range(4):
mixed = reduced * post[:, target : target + 1]
for source in range(4):
mixed = torch.addcmul(
mixed, residual[:, source].float(), comb[:, source, target, None]
)
output[:, target] = mixed.bfloat16()
collapse = torch.addcmul(
collapse, output[:, target].float(), pre[:, target : target + 1]
)
prenorm = collapse.bfloat16().float()
inv_rms = torch.rsqrt(prenorm.square().mean(-1, keepdim=True) + eps)
return output, (prenorm * inv_rms * weight.float()).bfloat16()
@ray.remote(num_gpus=1, max_calls=1)
def _all_reduce_mhc(monkeypatch, tp_size, pp_size, rank, distributed_init_port):
from vllm.model_executor.kernels.mhc.tilelang import (
mhc_post_tilelang,
mhc_pre_delayed_tilelang,
)
from vllm.models.deepseek_v41.nvidia.ops.cute_dsl import AllReduceMHC
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
@@ -38,98 +69,114 @@ def _all_reduce_mhc(monkeypatch, tp_size, pp_size, rank, distributed_init_port):
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
comm = get_tp_group().device_communicator.ca_comm
assert comm is not None and comm.mnnvl_lamport_ag_multicast_ptr
fn = torch.zeros(24, 20480, device=device)
scale = torch.ones(3, device=device)
base = torch.zeros(24, device=device)
op = AllReduceMHC(
hidden_size=5120, hc_mult=4, max_num_tokens=16, top_k=6, device=device
)
# Changing shapes and interleaving all-gather exercise the shared
# Lamport stages, including cleanup of a larger previous payload.
def run(n):
torch.manual_seed(42 + rank)
x = torch.randn(n, 5120, device=device, dtype=torch.bfloat16)
# Packed +0/-0 pairs collide with the Lamport sentinel.
x[:, :16] = 0
x[:, 9:16:2] = -0.0
def check_eager_and_replayed(fused, check, halved, doubled):
check(fused())
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
for _ in range(5):
fused()
captured = fused()
for _ in range(20):
graph.replay()
check(captured)
# Replays must pick up in-place input changes.
halved.mul_(0.5)
doubled.mul_(2)
graph.replay()
check(captured)
def mhc_inputs(n):
torch.manual_seed(123)
residual = torch.randn(n, 4, 5120, device=device, dtype=torch.bfloat16)
post = torch.rand(n, 4, device=device)
comb = torch.randn(n, 4, 4, device=device) * 0.1
pre = torch.rand(n, 4, device=device)
weight = torch.randn(5120, device=device, dtype=torch.bfloat16)
output = torch.empty_like(residual)
normalized = torch.empty_like(x)
return residual, post, comb, pre, weight
def run(n):
torch.manual_seed(42 + rank)
x = torch.randn(n, 5120, device=device, dtype=torch.bfloat16)
# Packed +0/-0 pairs collide with the Lamport sentinel.
x[:, :16] = 0
x[:, 9:16:2] = -0.0
residual, post, comb, pre, weight = mhc_inputs(n)
def fused():
torch.ops._C_custom_ar.all_reduce_mhc(
x,
return op(x, residual, post, comb, pre, weight, 1e-6)
def check(outputs):
peers = get_tp_group().all_gather(x, dim=0).view(tp_size, n, 5120)
output, normalized = _unfused_all_reduce_mhc(
peers, residual, post, comb, pre, weight, 1e-6
)
# The mixed hc streams match bit for bit. The RMSNorm sums the
# squares in another order and uses an approximate rsqrt.
assert torch.equal(outputs[0], output)
assert _bf16_ulps(outputs[1], normalized) <= 1
check_eager_and_replayed(fused, check, x, residual)
def run_finalize(n):
torch.manual_seed(7 + rank)
# A padded permuted GEMM2 buffer, like the MoE's.
rows = n * 6 + 5
gemm2 = torch.randn(rows, 5120, device=device, dtype=torch.bfloat16)
permuted = torch.randperm(rows, device=device)[: n * 6].view(n, 6).int()
# A route to an expert this rank does not hold.
permuted[0, -1] = -1
weights = torch.rand(n, 6, device=device)
shared = torch.randn(n, 5120, device=device, dtype=torch.bfloat16)
residual, post, comb, pre, weight = mhc_inputs(n)
def fused():
return op.finalize(
gemm2,
weights,
permuted,
shared,
residual,
post,
comb,
pre,
weight,
output,
normalized,
comm.mnnvl_lamport_ag_local_ptr,
comm.mnnvl_lamport_ag_multicast_ptr,
comm.mnnvl_lamport_epochs[0],
rank,
comm.mnnvl_buffer_size,
1e-6,
)
def check():
gathered = comm.custom_all_gather(x)
assert gathered is not None
peers = gathered.view(tp_size, n, 5120).float()
reduced = peers[0].clone()
for peer in peers[1:]:
reduced.add_(peer)
expected = mhc_post_tilelang(
reduced.bfloat16(), residual, post.unsqueeze(-1), comb
)
expected_norm = mhc_pre_delayed_tilelang(
expected,
fn,
scale,
base,
1e-6,
1e-6,
1e-6,
2.0,
20,
pre_mix=pre,
norm_weight=weight,
)[2]
torch.testing.assert_close(output, expected, rtol=0.008, atol=1e-6)
torch.testing.assert_close(
normalized, expected_norm, rtol=0.008, atol=0.008
)
def check(outputs):
# One FP32 FMA per route in route order, the shared add, one
# rounding, then the plain path: only the finalize differs.
acc = torch.zeros(n, 5120, device=device)
for k in range(6):
valid = (permuted[:, k] >= 0).unsqueeze(-1)
rows_k = gemm2[permuted[:, k].clamp_min(0).long()].float()
torch.addcmul(
acc,
torch.where(valid, rows_k, 0.0),
torch.where(valid, weights[:, k : k + 1], 0.0),
out=acc,
)
x = (acc + shared.float()).bfloat16()
expected = op(x, residual, post, comb, pre, weight, 1e-6)
assert torch.equal(outputs[0], expected[0])
assert torch.equal(outputs[1], expected[1])
fused()
check()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
for _ in range(6):
fused()
for _ in range(20):
graph.replay()
check()
x.mul_(0.5)
residual.mul_(2)
graph.replay()
check()
check_eager_and_replayed(fused, check, gemm2, shared)
# Cover fixed Q6 and shrinking/growing adaptive verification batches.
# Changing shapes cover shrinking and growing batches in one mailbox.
for n in (1, 6, 12, 8, 16, 3, 5, 2, 4, 1):
run(n)
run_finalize(n)
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100), reason="Requires SM100"
)
def test_all_reduce_mhc_preserves_bf16_boundaries_and_graph_replay(monkeypatch):
def test_all_reduce_mhc_matches_unfused_path(monkeypatch):
if torch.accelerator.device_count() < 4:
pytest.skip("Requires four GPUs with NVLink multicast")
multi_process_parallel(monkeypatch, 4, 1, _all_reduce_mhc)