[Perf][MoE] Eliminate staging from NCCL symmetric reduce-scatter (#49194)
Signed-off-by: samnordmann <snordmann@nvidia.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Misha Goin
parent
bf049ddb8f
commit
4a99ca40d6
@@ -150,7 +150,9 @@ def _init_dp_environment(world_size, rank, port, dp_size, dp_port):
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _make_forward_context(rank, world_size, num_tokens_per_rank):
|
||||
def _make_forward_context(
|
||||
rank, world_size, num_tokens_per_rank, num_tokens_across_dp=None
|
||||
):
|
||||
"""Keep the vLLM config and mock-DP forward context active together.
|
||||
|
||||
Returns a context manager suitable for ``with`` statements.
|
||||
@@ -175,15 +177,15 @@ def _make_forward_context(rank, world_size, num_tokens_per_rank):
|
||||
is_moe_model=True,
|
||||
data_parallel_rank=rank,
|
||||
)
|
||||
if num_tokens_across_dp is None:
|
||||
num_tokens_across_dp = [num_tokens_per_rank] * world_size
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
set_forward_context(
|
||||
_AttnMeta(),
|
||||
vllm_config,
|
||||
num_tokens=num_tokens_per_rank,
|
||||
num_tokens_across_dp=torch.tensor(
|
||||
[num_tokens_per_rank] * world_size, dtype=torch.int
|
||||
),
|
||||
num_tokens_across_dp=torch.tensor(num_tokens_across_dp, dtype=torch.int),
|
||||
),
|
||||
):
|
||||
yield
|
||||
@@ -516,6 +518,121 @@ def test_one_sided_manager_workspace_grow(world_size):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeCombineGroup:
|
||||
def __init__(self, combine_input):
|
||||
self.combine_input = combine_input
|
||||
self.allocation_calls = 0
|
||||
self.combined_input = None
|
||||
|
||||
def allocate_combine_input(self, shape, dtype, device, **kwargs):
|
||||
self.allocation_calls += 1
|
||||
assert self.combine_input is not None
|
||||
assert self.combine_input.shape == shape
|
||||
assert self.combine_input.dtype == dtype
|
||||
assert self.combine_input.device == device
|
||||
return self.combine_input
|
||||
|
||||
def combine_into_output(self, hidden_states, output, **kwargs):
|
||||
self.combined_input = hidden_states
|
||||
output.copy_(hidden_states)
|
||||
return output
|
||||
|
||||
|
||||
def test_args_cpu_workspace_allocation(monkeypatch):
|
||||
from types import SimpleNamespace
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
kernel = object.__new__(mk.FusedMoEKernelModularImpl)
|
||||
kernel.fused_experts = SimpleNamespace(
|
||||
workspace_dtype=lambda dtype: dtype,
|
||||
workspace_shapes=lambda *args: ((0,), (0,), (2, 3)),
|
||||
finalize_weight_and_reduce_impl=lambda: None,
|
||||
)
|
||||
kernel.prepare_finalize = SimpleNamespace(
|
||||
allocate_fused_expert_output=lambda *args: None,
|
||||
)
|
||||
monkeypatch.setattr(mk.current_platform, "is_cpu", lambda: True)
|
||||
|
||||
workspace13, workspace2, output, specialized = kernel._allocate_buffers(
|
||||
torch.float32,
|
||||
torch.device("cpu"),
|
||||
2,
|
||||
2,
|
||||
3,
|
||||
3,
|
||||
1,
|
||||
1,
|
||||
1,
|
||||
None,
|
||||
MoEActivation.SILU,
|
||||
)
|
||||
assert workspace13.numel() == workspace2.numel() == 0
|
||||
assert output.shape == (2, 3)
|
||||
assert output.device.type == "cpu"
|
||||
assert not specialized
|
||||
|
||||
|
||||
def test_args_finalize_noop_reuses_fused_expert_output(monkeypatch):
|
||||
import vllm.model_executor.layers.fused_moe.prepare_finalize.naive_dp_ep as pf
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
|
||||
fused_expert_output = torch.arange(6, dtype=torch.float32).view(2, 3)
|
||||
output = torch.empty_like(fused_expert_output)
|
||||
group = _FakeCombineGroup(combine_input=None)
|
||||
monkeypatch.setattr(pf, "get_ep_group", lambda: group)
|
||||
|
||||
impl = pf.MoEPrepareAndFinalizeNaiveDPEPModular()
|
||||
impl.finalize(
|
||||
output,
|
||||
fused_expert_output,
|
||||
torch.ones((2, 1)),
|
||||
torch.zeros((2, 1), dtype=torch.long),
|
||||
False,
|
||||
TopKWeightAndReduceNoOP(),
|
||||
)
|
||||
|
||||
assert group.allocation_calls == 0
|
||||
assert group.combined_input is fused_expert_output
|
||||
torch.testing.assert_close(output, fused_expert_output)
|
||||
|
||||
|
||||
def test_args_finalize_contiguous_writes_combine_input(monkeypatch):
|
||||
import vllm.model_executor.layers.fused_moe.prepare_finalize.naive_dp_ep as pf
|
||||
import vllm.model_executor.layers.fused_moe.topk_weight_and_reduce as wr
|
||||
|
||||
fused_expert_output = torch.arange(12, dtype=torch.float32).view(2, 2, 3)
|
||||
original = fused_expert_output.clone()
|
||||
topk_weights = torch.tensor([[0.25, 0.75], [0.4, 0.6]])
|
||||
combine_input = torch.empty((2, 3), dtype=torch.float32)
|
||||
output = torch.empty_like(combine_input)
|
||||
group = _FakeCombineGroup(combine_input)
|
||||
monkeypatch.setattr(pf, "get_ep_group", lambda: group)
|
||||
monkeypatch.setattr(
|
||||
wr.ops,
|
||||
"moe_sum",
|
||||
lambda hidden_states, out: out.copy_(hidden_states.sum(dim=1)),
|
||||
)
|
||||
|
||||
impl = pf.MoEPrepareAndFinalizeNaiveDPEPModular()
|
||||
impl.finalize(
|
||||
output,
|
||||
fused_expert_output,
|
||||
topk_weights,
|
||||
torch.zeros((2, 2), dtype=torch.long),
|
||||
False,
|
||||
wr.TopKWeightAndReduceContiguous(),
|
||||
)
|
||||
|
||||
expected = (original * topk_weights.unsqueeze(-1)).sum(dim=1)
|
||||
assert group.allocation_calls == 1
|
||||
assert group.combined_input is combine_input
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
|
||||
def _args_dispatch_combine_worker(rank, world_size):
|
||||
from vllm.distributed.device_communicators.all2all import AgRsAll2AllManager
|
||||
from vllm.forward_context import get_forward_context
|
||||
@@ -604,14 +721,39 @@ def _args_dispatch_combine_worker(rank, world_size):
|
||||
# -- combine (reduce-scatter) --
|
||||
# Each token i has value i in all columns; after reduce-scatter
|
||||
# each rank gets its slice, summed across ranks.
|
||||
expert_out = (
|
||||
expert_values = (
|
||||
torch.arange(total_tokens, device=device, dtype=torch.float32)
|
||||
.unsqueeze(1)
|
||||
.expand(total_tokens, hidden_size)
|
||||
.contiguous()
|
||||
)
|
||||
expert_out = manager.allocate_combine_input(
|
||||
expert_values.shape,
|
||||
expert_values.dtype,
|
||||
expert_values.device,
|
||||
is_sequence_parallel=True,
|
||||
)
|
||||
if expert_out is None:
|
||||
expert_out = expert_values
|
||||
else:
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import (
|
||||
is_symmetric_memory_tensor,
|
||||
)
|
||||
|
||||
combined = manager.combine(expert_out, is_sequence_parallel=True)
|
||||
assert is_symmetric_memory_tensor(expert_out)
|
||||
expert_out.copy_(expert_values)
|
||||
|
||||
combine_output = torch.empty(
|
||||
(tokens_per_rank, hidden_size),
|
||||
device=device,
|
||||
dtype=expert_out.dtype,
|
||||
)
|
||||
combined = manager.combine_into_output(
|
||||
expert_out,
|
||||
combine_output,
|
||||
is_sequence_parallel=True,
|
||||
)
|
||||
assert combined.data_ptr() == combine_output.data_ptr()
|
||||
assert combined.shape == (tokens_per_rank, hidden_size)
|
||||
|
||||
for i in range(tokens_per_rank):
|
||||
@@ -623,6 +765,61 @@ def _args_dispatch_combine_worker(rank, world_size):
|
||||
|
||||
torch.distributed.barrier()
|
||||
|
||||
uneven_sizes = [tokens_per_rank + r for r in range(world_size)]
|
||||
with _make_forward_context(
|
||||
rank,
|
||||
world_size,
|
||||
uneven_sizes[rank],
|
||||
num_tokens_across_dp=uneven_sizes,
|
||||
):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
with dp_metadata.sp_local_sizes(sequence_parallel_size=1):
|
||||
assert (
|
||||
manager.allocate_combine_input(
|
||||
(sum(uneven_sizes), hidden_size),
|
||||
torch.float32,
|
||||
device,
|
||||
is_sequence_parallel=True,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
total_uneven_tokens = sum(uneven_sizes)
|
||||
uneven_input = (
|
||||
torch.arange(
|
||||
total_uneven_tokens,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
.unsqueeze(1)
|
||||
.expand(total_uneven_tokens, hidden_size)
|
||||
.contiguous()
|
||||
)
|
||||
uneven_input.add_(rank)
|
||||
uneven_output = torch.empty(
|
||||
(uneven_sizes[rank], hidden_size),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
combined = get_ep_group().reduce_scatterv_into_output(
|
||||
uneven_input,
|
||||
uneven_output,
|
||||
dim=0,
|
||||
sizes=uneven_sizes,
|
||||
)
|
||||
assert combined.data_ptr() == uneven_output.data_ptr()
|
||||
|
||||
start = sum(uneven_sizes[:rank])
|
||||
expected = torch.arange(
|
||||
start,
|
||||
start + uneven_sizes[rank],
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
) * world_size + sum(range(world_size))
|
||||
expected = expected.unsqueeze(1).expand_as(combined)
|
||||
torch.testing.assert_close(combined, expected)
|
||||
|
||||
|
||||
@requires_multi_gpu
|
||||
@pytest.mark.parametrize("world_size", [2])
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import random
|
||||
import typing
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -12,11 +13,15 @@ import torch.multiprocessing as mp
|
||||
import vllm.envs as envs
|
||||
from tests.utils import ensure_current_vllm_config
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
|
||||
from vllm.distributed.device_communicators.cuda_communicator import (
|
||||
NCCL_DIRECT_SYMM_RS_OUTPUT_MIN_VERSION,
|
||||
CudaCommunicator,
|
||||
)
|
||||
from vllm.distributed.device_communicators.pynccl import register_nccl_symmetric_ops
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import (
|
||||
get_nccl_mem_pool,
|
||||
is_symmetric_memory_enabled,
|
||||
is_symmetric_memory_tensor,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tp_group,
|
||||
@@ -32,6 +37,53 @@ random.seed(44)
|
||||
test_size_elements = 4 * 1024 * 1024
|
||||
|
||||
|
||||
def test_disabled_pynccl_does_not_allocate_symmetric_buffer():
|
||||
communicator = object.__new__(CudaCommunicator)
|
||||
communicator.pynccl_comm = SimpleNamespace(disabled=True)
|
||||
|
||||
assert (
|
||||
communicator.get_symmetric_memory_buffer(
|
||||
"test", (1,), torch.float32, torch.device("cuda")
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_aiter", [False, True])
|
||||
def test_reduce_scatterv_preserves_caller_output(monkeypatch, use_aiter):
|
||||
import vllm.distributed.device_communicators.cuda_communicator as cc
|
||||
|
||||
communicator = object.__new__(CudaCommunicator)
|
||||
communicator.world_size = 2
|
||||
communicator.rank_in_group = 0
|
||||
input_ = torch.arange(12, dtype=torch.float32).view(4, 3)
|
||||
output = torch.full((2, 3), float("nan"))
|
||||
|
||||
def reduce_scatter(out, inp):
|
||||
assert out is output
|
||||
out.copy_(inp.view(2, 2, 3).sum(dim=0))
|
||||
|
||||
communicator.pynccl_comm = SimpleNamespace(
|
||||
disabled=False, nccl_version=22902, reduce_scatter=reduce_scatter
|
||||
)
|
||||
communicator._can_use_aiter_ag_rs = lambda sizes: use_aiter and sizes is None
|
||||
communicator.aiter_ar_comm = SimpleNamespace(
|
||||
should_custom_rs=lambda inp, dim: True,
|
||||
custom_reduce_scatter=lambda inp, out, dim: reduce_scatter(out, inp),
|
||||
)
|
||||
monkeypatch.setattr(cc, "should_nccl_symm_mem_ag_rs", lambda: True)
|
||||
monkeypatch.setattr(cc, "is_symmetric_memory_tensor", lambda tensor: False)
|
||||
monkeypatch.setenv("VLLM_BATCH_INVARIANT", "0")
|
||||
|
||||
# Explicit uniform sizes must not stage an ordinary input into symmetric
|
||||
# scratch, and AITER must write the supplied output after normalization.
|
||||
result = communicator.reduce_scatterv_into_output(
|
||||
input_, output, dim=0, sizes=[2, 2]
|
||||
)
|
||||
assert result.data_ptr() == output.data_ptr()
|
||||
torch.testing.assert_close(output, input_.view(2, 2, 3).sum(dim=0))
|
||||
|
||||
|
||||
def nccl_symm_mem_allreduce_worker(local_rank: int, world_size: int):
|
||||
monkeypatch = pytest.MonkeyPatch()
|
||||
with monkeypatch.context() as m:
|
||||
@@ -193,15 +245,107 @@ def nccl_symm_mem_reduce_scatter_worker(local_rank: int, world_size: int):
|
||||
pytest.skip("NCCL symmetric memory is disabled.")
|
||||
|
||||
per_rank_size = test_size_elements // world_size
|
||||
input_tensor = torch.randint(
|
||||
ordinary_input = torch.randint(
|
||||
1, 23, (test_size_elements,), dtype=dtype, device=device
|
||||
)
|
||||
input_clone = input_tensor.clone()
|
||||
output = cuda_communicator.reduce_scatter(input_tensor, dim=0)
|
||||
ordinary_clone = ordinary_input.clone()
|
||||
ordinary_output = cuda_communicator.reduce_scatter(ordinary_input, dim=0)
|
||||
|
||||
group = get_tp_group().device_group
|
||||
expected = torch.empty(per_rank_size, dtype=dtype, device=device)
|
||||
ordinary_expected = torch.empty(per_rank_size, dtype=dtype, device=device)
|
||||
dist.reduce_scatter_tensor(ordinary_expected, ordinary_clone, group=group)
|
||||
torch.testing.assert_close(
|
||||
ordinary_output, ordinary_expected, atol=2.5, rtol=0.1
|
||||
)
|
||||
assert is_symmetric_memory_tensor(ordinary_output)
|
||||
|
||||
# Both calls reuse rs_out; validate the first result before overwriting it.
|
||||
for sizes in (None, [per_rank_size] * world_size):
|
||||
ordinary_v_output = cuda_communicator.reduce_scatterv(
|
||||
ordinary_input, dim=0, sizes=sizes
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
ordinary_v_output, ordinary_expected, atol=2.5, rtol=0.1
|
||||
)
|
||||
assert is_symmetric_memory_tensor(ordinary_v_output)
|
||||
|
||||
pynccl_comm = cuda_communicator.pynccl_comm
|
||||
assert pynccl_comm is not None
|
||||
if pynccl_comm.nccl_version < NCCL_DIRECT_SYMM_RS_OUTPUT_MIN_VERSION:
|
||||
return
|
||||
|
||||
from vllm.v1.worker import ubatching
|
||||
|
||||
m.setattr(ubatching, "dbo_current_ubatch_id", lambda: 0)
|
||||
ubatch0 = cuda_communicator._get_symm_scratch("ag_out", (128,), dtype, device)
|
||||
ubatch0.fill_(1)
|
||||
m.setattr(ubatching, "dbo_current_ubatch_id", lambda: 1)
|
||||
ubatch1 = cuda_communicator._get_symm_scratch("ag_out", (128,), dtype, device)
|
||||
ubatch1.fill_(2)
|
||||
assert ubatch0.data_ptr() != ubatch1.data_ptr()
|
||||
assert (ubatch0 == 1).all()
|
||||
|
||||
m.setattr(ubatching, "dbo_current_ubatch_id", lambda: 0)
|
||||
smaller = cuda_communicator._get_symm_scratch("ag_out", (64,), dtype, device)
|
||||
assert smaller.shape == (64,)
|
||||
assert smaller.data_ptr() == ubatch0.data_ptr()
|
||||
|
||||
grown = cuda_communicator._get_symm_scratch("ag_out", (129,), dtype, device)
|
||||
assert grown.shape == (129,)
|
||||
assert grown.data_ptr() != ubatch0.data_ptr()
|
||||
reused = cuda_communicator._get_symm_scratch("ag_out", (200,), dtype, device)
|
||||
assert reused.data_ptr() == grown.data_ptr()
|
||||
|
||||
input_tensor = cuda_communicator._get_symm_scratch(
|
||||
"test_graph_rs_input", (test_size_elements,), dtype, device
|
||||
)
|
||||
input_tensor.random_(1, 23)
|
||||
input_clone = input_tensor.clone()
|
||||
output = torch.empty(per_rank_size, dtype=dtype, device=device)
|
||||
assert is_symmetric_memory_tensor(input_tensor)
|
||||
assert not is_symmetric_memory_tensor(output)
|
||||
|
||||
result = cuda_communicator.reduce_scatterv_into_output(
|
||||
input_tensor,
|
||||
output,
|
||||
dim=0,
|
||||
sizes=[per_rank_size] * world_size,
|
||||
)
|
||||
assert result.data_ptr() == output.data_ptr()
|
||||
|
||||
expected = torch.empty_like(output)
|
||||
dist.reduce_scatter_tensor(expected, input_clone, group=group)
|
||||
torch.testing.assert_close(result, expected, atol=2.5, rtol=0.1)
|
||||
|
||||
dist.barrier(group=get_tp_group().cpu_group)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
graph_result = cuda_communicator.reduce_scatterv_into_output(
|
||||
input_tensor,
|
||||
output,
|
||||
dim=0,
|
||||
sizes=[per_rank_size] * world_size,
|
||||
)
|
||||
assert graph_result.data_ptr() == output.data_ptr()
|
||||
|
||||
input_tensor.random_(24, 47)
|
||||
input_clone.copy_(input_tensor)
|
||||
dist.reduce_scatter_tensor(expected, input_clone, group=group)
|
||||
captured_input_ptr = input_tensor.data_ptr()
|
||||
del input_tensor
|
||||
grown_input = cuda_communicator._get_symm_scratch(
|
||||
"test_graph_rs_input", (test_size_elements + 1,), dtype, device
|
||||
)
|
||||
assert grown_input.data_ptr() != captured_input_ptr
|
||||
retired = cuda_communicator.__dict__["_retired_symm_scratch_bufs"]
|
||||
assert any(
|
||||
buf.data_ptr() == captured_input_ptr
|
||||
for buffers in retired.values()
|
||||
for buf in buffers
|
||||
)
|
||||
output.zero_()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(output, expected, atol=2.5, rtol=0.1)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user