[Perf][Distributed] Add low-SM multimem reduce-scatter for SM100/SM103 (#55072)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
Yongye Zhu
2026-09-25 03:27:21 +00:00
committed by GitHub
co-authored by Summer Yang
parent 8b365ff949
commit 442bd00d9f
10 changed files with 1243 additions and 31 deletions
@@ -23,6 +23,7 @@ import torch
import torch.distributed as dist
from vllm.distributed.parallel_state import get_tp_group
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter
from vllm.platforms import current_platform
from ..utils import (
@@ -53,6 +54,11 @@ def _supports_multimem():
)
def _supports_sm100_or_sm103():
capability = current_platform.get_device_capability()
return current_platform.is_cuda() and capability in ((10, 0), (10, 3))
def _payload(value, device):
return torch.full((_ROWS, _HIDDEN), value, dtype=torch.bfloat16, device=device)
@@ -324,3 +330,65 @@ def test_mnnvl_lamport_stage_cleanup(monkeypatch: pytest.MonkeyPatch):
if torch.accelerator.device_count() < 2:
pytest.skip("Need at least two GPUs to run the test.")
multi_process_parallel(monkeypatch, 2, 1, _run_stage_cleanup_test)
@ray.remote(num_gpus=1, max_calls=1)
def _run_multimem_reduce_scatter_test(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
fa = get_tp_group().device_communicator.ca_comm
assert fa is not None and not fa.disabled
assert fa.mnnvl_multimem_rs_supported
assert not fa.mnnvl_multimem_rs_initialized
assert not fa.mnnvl_multimem_rs_multicast_ptr
message_bytes = {
"mnnvl_lamport": 1 * 1024 * 1024,
"mnnvl_multimem": 20 * 1024 * 1024,
"nccl": 64 * 1024 * 1024 + 128,
}
for _ in range(2):
for backend, size_bytes in message_bytes.items():
shard_elements = size_bytes // torch.bfloat16.itemsize // tp_size
inp = torch.full(
(tp_size, shard_elements),
rank + 1,
dtype=torch.bfloat16,
device=device,
)
selected = fa._select_reduce_scatter_backend(inp)
if backend == "nccl":
assert selected is None
out = sp_reduce_scatter(inp)
else:
assert selected == backend
out = fa.custom_reduce_scatter(inp)
assert out is not None
if backend == "mnnvl_multimem":
assert fa.mnnvl_multimem_rs_initialized
assert fa.mnnvl_multimem_rs_multicast_ptr
expected = tp_size * (tp_size + 1) // 2
torch.testing.assert_close(out, torch.full_like(out, expected))
@pytest.mark.skipif(
not _supports_sm100_or_sm103(),
reason="The low-SM MNNVL reduce-scatter path requires SM100 or SM103.",
)
@pytest.mark.parametrize("tp_size", [2, 4, 8])
def test_mnnvl_multimem_reduce_scatter(monkeypatch: pytest.MonkeyPatch, tp_size: int):
if torch.accelerator.device_count() < tp_size:
pytest.skip(f"Need at least {tp_size} GPUs to run the test.")
multi_process_parallel(monkeypatch, tp_size, 1, _run_multimem_reduce_scatter_test)
+263 -1
View File
@@ -151,9 +151,9 @@ def test_custom_allreduce_filters_dtype(
) -> None:
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator._ptr = 0
communicator.world_size = 2
communicator.max_size = 1024
communicator._ptr = 0
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
@@ -219,6 +219,268 @@ def test_local_multicast_support_rejects_non_cuda(monkeypatch):
assert not car._has_local_multicast_support(torch.device("cuda:0"))
@pytest.mark.parametrize(
("world_size", "device_capability", "local_multicast", "expected"),
[
(2, (10, 0), True, True),
(4, (10, 3), True, True),
(8, (10, 0), True, True),
(8, (10, 3), True, True),
(6, (10, 3), True, False),
(8, (10, 1), True, False),
(8, (9, 0), True, False),
(8, (10, 3), False, False),
],
)
def test_mnnvl_multimem_reduce_scatter_platform_gate(
monkeypatch,
world_size,
device_capability,
local_multicast,
expected,
):
def is_device_capability(capability, device_id):
assert capability in ((10, 0), (10, 3))
assert device_id == 3
return device_capability == capability
monkeypatch.setattr(
car.current_platform,
"is_device_capability",
is_device_capability,
)
monkeypatch.setattr(
car,
"_has_local_multicast_support",
lambda _device: local_multicast,
)
supported = car._supports_mnnvl_multimem_reduce_scatter(
torch.device("cuda:3"), world_size
)
assert supported is expected
@pytest.mark.parametrize(
(
"message_bytes",
"multimem_ptr",
"multimem_initialized",
"batch_invariant",
"expected",
),
[
(16 * 1024 * 1024, 1, True, False, "mnnvl_lamport"),
(16 * 1024 * 1024 + 128, 1, True, False, "mnnvl_multimem"),
(64 * 1024 * 1024, 1, True, False, "mnnvl_multimem"),
(64 * 1024 * 1024 + 128, 1, True, False, None),
(32 * 1024 * 1024, 0, True, False, None),
(32 * 1024 * 1024, 0, False, False, "mnnvl_multimem"),
(8 * 1024 * 1024, 1, True, True, "mnnvl_lamport"),
(32 * 1024 * 1024, 1, True, True, None),
],
)
@pytest.mark.parametrize("world_size", [2, 4, 8])
def test_mnnvl_reduce_scatter_backend_gate(
monkeypatch,
world_size,
message_bytes,
multimem_ptr,
multimem_initialized,
batch_invariant,
expected,
):
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(car.envs, "VLLM_BATCH_INVARIANT", batch_invariant)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator._ptr = 0
communicator.world_size = world_size
communicator.mnnvl_only = False
communicator.fully_connected = True
communicator.mnnvl_multicast_ptr = 1
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = multimem_initialized
communicator.mnnvl_multimem_rs_multicast_ptr = multimem_ptr
communicator.max_mnnvl_reduce_scatter_size = 16 * 1024 * 1024
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
communicator.max_reduce_scatter_size = 16 * 1024 * 1024
inp = torch.empty(
(world_size, message_bytes // torch.bfloat16.itemsize // world_size),
dtype=torch.bfloat16,
)
assert inp.nbytes == message_bytes
assert communicator._select_reduce_scatter_backend(inp) == expected
assert communicator.should_custom_reduce_scatter(inp) is (expected is not None)
assert communicator.should_mnnvl_multimem_reduce_scatter(inp) is (
expected == "mnnvl_multimem"
)
def test_mnnvl_multimem_reduce_scatter_skips_rendezvous_after_peer_alloc_failure(
monkeypatch,
):
events = []
class FakeSymmMem:
@staticmethod
def empty(*_args, **_kwargs):
events.append("empty")
return torch.empty(1, dtype=torch.uint8)
@staticmethod
def rendezvous(*_args, **_kwargs):
events.append("rendezvous")
return None
def report_peer_allocation_failure(group_value, **_kwargs):
events.append("all_reduce")
assert group_value.item() == 1
group_value.zero_()
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
monkeypatch.setattr(car.dist, "all_reduce", report_peer_allocation_failure)
warnings = []
monkeypatch.setattr(
car.logger,
"warning_once",
lambda message, *_args, **_kwargs: warnings.append(message),
)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = True
communicator._ptr = 0
communicator.group = object()
communicator.device = torch.device("cuda:0")
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = False
communicator.mnnvl_multimem_rs_buffer = None
communicator.mnnvl_multimem_rs_multicast_ptr = 0
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
assert events == ["empty", "all_reduce"]
assert communicator.mnnvl_multimem_rs_initialized
assert communicator.mnnvl_multimem_rs_buffer is None
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
assert warnings == [
"MNNVL multimem reduce-scatter symmetric-memory allocation "
"failed on at least one rank; falling back to NCCL."
]
def test_mnnvl_multimem_reduce_scatter_warns_on_rendezvous_failure(monkeypatch):
events = []
class FakeSymmMem:
@staticmethod
def empty(*_args, **_kwargs):
events.append("empty")
return torch.empty(1, dtype=torch.uint8)
@staticmethod
def rendezvous(*_args, **_kwargs):
events.append("rendezvous")
raise RuntimeError("rendezvous failed")
def preserve_local_result(_group_value, **_kwargs):
events.append("all_reduce")
warnings = []
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
monkeypatch.setattr(
car.logger,
"warning_once",
lambda message, *_args, **_kwargs: warnings.append(message),
)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = True
communicator._ptr = 0
communicator.group = type("Group", (), {"group_name": "test"})()
communicator.device = torch.device("cuda:0")
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = False
communicator.mnnvl_multimem_rs_buffer = None
communicator.mnnvl_multimem_rs_multicast_ptr = 0
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
assert events == ["empty", "all_reduce", "rendezvous", "all_reduce"]
assert communicator.mnnvl_multimem_rs_initialized
assert communicator.mnnvl_multimem_rs_buffer is None
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
assert warnings == [
"MNNVL multimem reduce-scatter symmetric-memory rendezvous "
"failed on at least one rank; falling back to NCCL."
]
def test_mnnvl_multimem_reduce_scatter_initializes_signals(monkeypatch):
events = []
buffers = []
class FakeHandle:
multicast_ptr = 0x3000
class FakeSymmMem:
@staticmethod
def empty(size, **_kwargs):
events.append(("empty", size))
buffer = torch.ones(size, dtype=torch.uint8)
buffers.append(buffer)
return buffer
@staticmethod
def rendezvous(*_args, **_kwargs):
events.append(("rendezvous", None))
return FakeHandle()
def preserve_local_result(*_args, **_kwargs):
events.append(("all_reduce", None))
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
monkeypatch.setattr(
car.torch.accelerator,
"synchronize",
lambda: events.append(("synchronize", None)),
)
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = True
communicator._ptr = 0
communicator.group = type("Group", (), {"group_name": "test"})()
communicator.device = torch.device("cpu")
communicator.max_mnnvl_multimem_reduce_scatter_size = 129
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = False
communicator.mnnvl_multimem_rs_buffer = None
communicator.mnnvl_multimem_rs_multicast_ptr = 0
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
assert events == [
("empty", 257),
("all_reduce", None),
("rendezvous", None),
("synchronize", None),
("all_reduce", None),
]
assert torch.all(buffers[0][:128] == 0)
assert torch.all(buffers[0][128:] == 1)
assert communicator.mnnvl_multimem_rs_buffer_size == 129
assert communicator.mnnvl_multimem_rs_local_ptr == buffers[0].data_ptr() + 128
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0x3080
@ray.remote(num_gpus=1, max_calls=1)
def graph_allreduce(
monkeypatch: pytest.MonkeyPatch,