[Perf][MoE] Eliminate staging from NCCL symmetric reduce-scatter (#49194)

Signed-off-by: samnordmann <snordmann@nvidia.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Samuel Nordmann
2026-10-06 07:37:01 -07:00
committed by GitHub
co-authored by Misha Goin
parent bf049ddb8f
commit 4a99ca40d6
8 changed files with 723 additions and 69 deletions
+203 -6
View File
@@ -150,7 +150,9 @@ def _init_dp_environment(world_size, rank, port, dp_size, dp_port):
@contextmanager
def _make_forward_context(rank, world_size, num_tokens_per_rank):
def _make_forward_context(
rank, world_size, num_tokens_per_rank, num_tokens_across_dp=None
):
"""Keep the vLLM config and mock-DP forward context active together.
Returns a context manager suitable for ``with`` statements.
@@ -175,15 +177,15 @@ def _make_forward_context(rank, world_size, num_tokens_per_rank):
is_moe_model=True,
data_parallel_rank=rank,
)
if num_tokens_across_dp is None:
num_tokens_across_dp = [num_tokens_per_rank] * world_size
with (
set_current_vllm_config(vllm_config),
set_forward_context(
_AttnMeta(),
vllm_config,
num_tokens=num_tokens_per_rank,
num_tokens_across_dp=torch.tensor(
[num_tokens_per_rank] * world_size, dtype=torch.int
),
num_tokens_across_dp=torch.tensor(num_tokens_across_dp, dtype=torch.int),
),
):
yield
@@ -516,6 +518,121 @@ def test_one_sided_manager_workspace_grow(world_size):
# ---------------------------------------------------------------------------
class _FakeCombineGroup:
def __init__(self, combine_input):
self.combine_input = combine_input
self.allocation_calls = 0
self.combined_input = None
def allocate_combine_input(self, shape, dtype, device, **kwargs):
self.allocation_calls += 1
assert self.combine_input is not None
assert self.combine_input.shape == shape
assert self.combine_input.dtype == dtype
assert self.combine_input.device == device
return self.combine_input
def combine_into_output(self, hidden_states, output, **kwargs):
self.combined_input = hidden_states
output.copy_(hidden_states)
return output
def test_args_cpu_workspace_allocation(monkeypatch):
from types import SimpleNamespace
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
kernel = object.__new__(mk.FusedMoEKernelModularImpl)
kernel.fused_experts = SimpleNamespace(
workspace_dtype=lambda dtype: dtype,
workspace_shapes=lambda *args: ((0,), (0,), (2, 3)),
finalize_weight_and_reduce_impl=lambda: None,
)
kernel.prepare_finalize = SimpleNamespace(
allocate_fused_expert_output=lambda *args: None,
)
monkeypatch.setattr(mk.current_platform, "is_cpu", lambda: True)
workspace13, workspace2, output, specialized = kernel._allocate_buffers(
torch.float32,
torch.device("cpu"),
2,
2,
3,
3,
1,
1,
1,
None,
MoEActivation.SILU,
)
assert workspace13.numel() == workspace2.numel() == 0
assert output.shape == (2, 3)
assert output.device.type == "cpu"
assert not specialized
def test_args_finalize_noop_reuses_fused_expert_output(monkeypatch):
import vllm.model_executor.layers.fused_moe.prepare_finalize.naive_dp_ep as pf
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
fused_expert_output = torch.arange(6, dtype=torch.float32).view(2, 3)
output = torch.empty_like(fused_expert_output)
group = _FakeCombineGroup(combine_input=None)
monkeypatch.setattr(pf, "get_ep_group", lambda: group)
impl = pf.MoEPrepareAndFinalizeNaiveDPEPModular()
impl.finalize(
output,
fused_expert_output,
torch.ones((2, 1)),
torch.zeros((2, 1), dtype=torch.long),
False,
TopKWeightAndReduceNoOP(),
)
assert group.allocation_calls == 0
assert group.combined_input is fused_expert_output
torch.testing.assert_close(output, fused_expert_output)
def test_args_finalize_contiguous_writes_combine_input(monkeypatch):
import vllm.model_executor.layers.fused_moe.prepare_finalize.naive_dp_ep as pf
import vllm.model_executor.layers.fused_moe.topk_weight_and_reduce as wr
fused_expert_output = torch.arange(12, dtype=torch.float32).view(2, 2, 3)
original = fused_expert_output.clone()
topk_weights = torch.tensor([[0.25, 0.75], [0.4, 0.6]])
combine_input = torch.empty((2, 3), dtype=torch.float32)
output = torch.empty_like(combine_input)
group = _FakeCombineGroup(combine_input)
monkeypatch.setattr(pf, "get_ep_group", lambda: group)
monkeypatch.setattr(
wr.ops,
"moe_sum",
lambda hidden_states, out: out.copy_(hidden_states.sum(dim=1)),
)
impl = pf.MoEPrepareAndFinalizeNaiveDPEPModular()
impl.finalize(
output,
fused_expert_output,
topk_weights,
torch.zeros((2, 2), dtype=torch.long),
False,
wr.TopKWeightAndReduceContiguous(),
)
expected = (original * topk_weights.unsqueeze(-1)).sum(dim=1)
assert group.allocation_calls == 1
assert group.combined_input is combine_input
torch.testing.assert_close(output, expected)
def _args_dispatch_combine_worker(rank, world_size):
from vllm.distributed.device_communicators.all2all import AgRsAll2AllManager
from vllm.forward_context import get_forward_context
@@ -604,14 +721,39 @@ def _args_dispatch_combine_worker(rank, world_size):
# -- combine (reduce-scatter) --
# Each token i has value i in all columns; after reduce-scatter
# each rank gets its slice, summed across ranks.
expert_out = (
expert_values = (
torch.arange(total_tokens, device=device, dtype=torch.float32)
.unsqueeze(1)
.expand(total_tokens, hidden_size)
.contiguous()
)
expert_out = manager.allocate_combine_input(
expert_values.shape,
expert_values.dtype,
expert_values.device,
is_sequence_parallel=True,
)
if expert_out is None:
expert_out = expert_values
else:
from vllm.distributed.device_communicators.pynccl_allocator import (
is_symmetric_memory_tensor,
)
combined = manager.combine(expert_out, is_sequence_parallel=True)
assert is_symmetric_memory_tensor(expert_out)
expert_out.copy_(expert_values)
combine_output = torch.empty(
(tokens_per_rank, hidden_size),
device=device,
dtype=expert_out.dtype,
)
combined = manager.combine_into_output(
expert_out,
combine_output,
is_sequence_parallel=True,
)
assert combined.data_ptr() == combine_output.data_ptr()
assert combined.shape == (tokens_per_rank, hidden_size)
for i in range(tokens_per_rank):
@@ -623,6 +765,61 @@ def _args_dispatch_combine_worker(rank, world_size):
torch.distributed.barrier()
uneven_sizes = [tokens_per_rank + r for r in range(world_size)]
with _make_forward_context(
rank,
world_size,
uneven_sizes[rank],
num_tokens_across_dp=uneven_sizes,
):
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
with dp_metadata.sp_local_sizes(sequence_parallel_size=1):
assert (
manager.allocate_combine_input(
(sum(uneven_sizes), hidden_size),
torch.float32,
device,
is_sequence_parallel=True,
)
is None
)
total_uneven_tokens = sum(uneven_sizes)
uneven_input = (
torch.arange(
total_uneven_tokens,
dtype=torch.float32,
device=device,
)
.unsqueeze(1)
.expand(total_uneven_tokens, hidden_size)
.contiguous()
)
uneven_input.add_(rank)
uneven_output = torch.empty(
(uneven_sizes[rank], hidden_size),
dtype=torch.float32,
device=device,
)
combined = get_ep_group().reduce_scatterv_into_output(
uneven_input,
uneven_output,
dim=0,
sizes=uneven_sizes,
)
assert combined.data_ptr() == uneven_output.data_ptr()
start = sum(uneven_sizes[:rank])
expected = torch.arange(
start,
start + uneven_sizes[rank],
dtype=torch.float32,
device=device,
) * world_size + sum(range(world_size))
expected = expected.unsqueeze(1).expand_as(combined)
torch.testing.assert_close(combined, expected)
@requires_multi_gpu
@pytest.mark.parametrize("world_size", [2])
+149 -5
View File
@@ -3,6 +3,7 @@
import random
import typing
from types import SimpleNamespace
import pytest
import torch
@@ -12,11 +13,15 @@ import torch.multiprocessing as mp
import vllm.envs as envs
from tests.utils import ensure_current_vllm_config
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
from vllm.distributed.device_communicators.cuda_communicator import (
NCCL_DIRECT_SYMM_RS_OUTPUT_MIN_VERSION,
CudaCommunicator,
)
from vllm.distributed.device_communicators.pynccl import register_nccl_symmetric_ops
from vllm.distributed.device_communicators.pynccl_allocator import (
get_nccl_mem_pool,
is_symmetric_memory_enabled,
is_symmetric_memory_tensor,
)
from vllm.distributed.parallel_state import (
get_tp_group,
@@ -32,6 +37,53 @@ random.seed(44)
test_size_elements = 4 * 1024 * 1024
def test_disabled_pynccl_does_not_allocate_symmetric_buffer():
communicator = object.__new__(CudaCommunicator)
communicator.pynccl_comm = SimpleNamespace(disabled=True)
assert (
communicator.get_symmetric_memory_buffer(
"test", (1,), torch.float32, torch.device("cuda")
)
is None
)
@pytest.mark.parametrize("use_aiter", [False, True])
def test_reduce_scatterv_preserves_caller_output(monkeypatch, use_aiter):
import vllm.distributed.device_communicators.cuda_communicator as cc
communicator = object.__new__(CudaCommunicator)
communicator.world_size = 2
communicator.rank_in_group = 0
input_ = torch.arange(12, dtype=torch.float32).view(4, 3)
output = torch.full((2, 3), float("nan"))
def reduce_scatter(out, inp):
assert out is output
out.copy_(inp.view(2, 2, 3).sum(dim=0))
communicator.pynccl_comm = SimpleNamespace(
disabled=False, nccl_version=22902, reduce_scatter=reduce_scatter
)
communicator._can_use_aiter_ag_rs = lambda sizes: use_aiter and sizes is None
communicator.aiter_ar_comm = SimpleNamespace(
should_custom_rs=lambda inp, dim: True,
custom_reduce_scatter=lambda inp, out, dim: reduce_scatter(out, inp),
)
monkeypatch.setattr(cc, "should_nccl_symm_mem_ag_rs", lambda: True)
monkeypatch.setattr(cc, "is_symmetric_memory_tensor", lambda tensor: False)
monkeypatch.setenv("VLLM_BATCH_INVARIANT", "0")
# Explicit uniform sizes must not stage an ordinary input into symmetric
# scratch, and AITER must write the supplied output after normalization.
result = communicator.reduce_scatterv_into_output(
input_, output, dim=0, sizes=[2, 2]
)
assert result.data_ptr() == output.data_ptr()
torch.testing.assert_close(output, input_.view(2, 2, 3).sum(dim=0))
def nccl_symm_mem_allreduce_worker(local_rank: int, world_size: int):
monkeypatch = pytest.MonkeyPatch()
with monkeypatch.context() as m:
@@ -193,15 +245,107 @@ def nccl_symm_mem_reduce_scatter_worker(local_rank: int, world_size: int):
pytest.skip("NCCL symmetric memory is disabled.")
per_rank_size = test_size_elements // world_size
input_tensor = torch.randint(
ordinary_input = torch.randint(
1, 23, (test_size_elements,), dtype=dtype, device=device
)
input_clone = input_tensor.clone()
output = cuda_communicator.reduce_scatter(input_tensor, dim=0)
ordinary_clone = ordinary_input.clone()
ordinary_output = cuda_communicator.reduce_scatter(ordinary_input, dim=0)
group = get_tp_group().device_group
expected = torch.empty(per_rank_size, dtype=dtype, device=device)
ordinary_expected = torch.empty(per_rank_size, dtype=dtype, device=device)
dist.reduce_scatter_tensor(ordinary_expected, ordinary_clone, group=group)
torch.testing.assert_close(
ordinary_output, ordinary_expected, atol=2.5, rtol=0.1
)
assert is_symmetric_memory_tensor(ordinary_output)
# Both calls reuse rs_out; validate the first result before overwriting it.
for sizes in (None, [per_rank_size] * world_size):
ordinary_v_output = cuda_communicator.reduce_scatterv(
ordinary_input, dim=0, sizes=sizes
)
torch.testing.assert_close(
ordinary_v_output, ordinary_expected, atol=2.5, rtol=0.1
)
assert is_symmetric_memory_tensor(ordinary_v_output)
pynccl_comm = cuda_communicator.pynccl_comm
assert pynccl_comm is not None
if pynccl_comm.nccl_version < NCCL_DIRECT_SYMM_RS_OUTPUT_MIN_VERSION:
return
from vllm.v1.worker import ubatching
m.setattr(ubatching, "dbo_current_ubatch_id", lambda: 0)
ubatch0 = cuda_communicator._get_symm_scratch("ag_out", (128,), dtype, device)
ubatch0.fill_(1)
m.setattr(ubatching, "dbo_current_ubatch_id", lambda: 1)
ubatch1 = cuda_communicator._get_symm_scratch("ag_out", (128,), dtype, device)
ubatch1.fill_(2)
assert ubatch0.data_ptr() != ubatch1.data_ptr()
assert (ubatch0 == 1).all()
m.setattr(ubatching, "dbo_current_ubatch_id", lambda: 0)
smaller = cuda_communicator._get_symm_scratch("ag_out", (64,), dtype, device)
assert smaller.shape == (64,)
assert smaller.data_ptr() == ubatch0.data_ptr()
grown = cuda_communicator._get_symm_scratch("ag_out", (129,), dtype, device)
assert grown.shape == (129,)
assert grown.data_ptr() != ubatch0.data_ptr()
reused = cuda_communicator._get_symm_scratch("ag_out", (200,), dtype, device)
assert reused.data_ptr() == grown.data_ptr()
input_tensor = cuda_communicator._get_symm_scratch(
"test_graph_rs_input", (test_size_elements,), dtype, device
)
input_tensor.random_(1, 23)
input_clone = input_tensor.clone()
output = torch.empty(per_rank_size, dtype=dtype, device=device)
assert is_symmetric_memory_tensor(input_tensor)
assert not is_symmetric_memory_tensor(output)
result = cuda_communicator.reduce_scatterv_into_output(
input_tensor,
output,
dim=0,
sizes=[per_rank_size] * world_size,
)
assert result.data_ptr() == output.data_ptr()
expected = torch.empty_like(output)
dist.reduce_scatter_tensor(expected, input_clone, group=group)
torch.testing.assert_close(result, expected, atol=2.5, rtol=0.1)
dist.barrier(group=get_tp_group().cpu_group)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_result = cuda_communicator.reduce_scatterv_into_output(
input_tensor,
output,
dim=0,
sizes=[per_rank_size] * world_size,
)
assert graph_result.data_ptr() == output.data_ptr()
input_tensor.random_(24, 47)
input_clone.copy_(input_tensor)
dist.reduce_scatter_tensor(expected, input_clone, group=group)
captured_input_ptr = input_tensor.data_ptr()
del input_tensor
grown_input = cuda_communicator._get_symm_scratch(
"test_graph_rs_input", (test_size_elements + 1,), dtype, device
)
assert grown_input.data_ptr() != captured_input_ptr
retired = cuda_communicator.__dict__["_retired_symm_scratch_bufs"]
assert any(
buf.data_ptr() == captured_input_ptr
for buffers in retired.values()
for buf in buffers
)
output.zero_()
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(output, expected, atol=2.5, rtol=0.1)