[Perf][Distributed] Add low-SM multimem reduce-scatter for SM100/SM103 (#55072)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
Yongye Zhu
2026-09-25 03:27:21 +00:00
committed by GitHub
co-authored by Summer Yang
parent 8b365ff949
commit 442bd00d9f
10 changed files with 1243 additions and 31 deletions
+263 -1
View File
@@ -151,9 +151,9 @@ def test_custom_allreduce_filters_dtype(
) -> None:
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator._ptr = 0
communicator.world_size = 2
communicator.max_size = 1024
communicator._ptr = 0
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
@@ -219,6 +219,268 @@ def test_local_multicast_support_rejects_non_cuda(monkeypatch):
assert not car._has_local_multicast_support(torch.device("cuda:0"))
@pytest.mark.parametrize(
("world_size", "device_capability", "local_multicast", "expected"),
[
(2, (10, 0), True, True),
(4, (10, 3), True, True),
(8, (10, 0), True, True),
(8, (10, 3), True, True),
(6, (10, 3), True, False),
(8, (10, 1), True, False),
(8, (9, 0), True, False),
(8, (10, 3), False, False),
],
)
def test_mnnvl_multimem_reduce_scatter_platform_gate(
monkeypatch,
world_size,
device_capability,
local_multicast,
expected,
):
def is_device_capability(capability, device_id):
assert capability in ((10, 0), (10, 3))
assert device_id == 3
return device_capability == capability
monkeypatch.setattr(
car.current_platform,
"is_device_capability",
is_device_capability,
)
monkeypatch.setattr(
car,
"_has_local_multicast_support",
lambda _device: local_multicast,
)
supported = car._supports_mnnvl_multimem_reduce_scatter(
torch.device("cuda:3"), world_size
)
assert supported is expected
@pytest.mark.parametrize(
(
"message_bytes",
"multimem_ptr",
"multimem_initialized",
"batch_invariant",
"expected",
),
[
(16 * 1024 * 1024, 1, True, False, "mnnvl_lamport"),
(16 * 1024 * 1024 + 128, 1, True, False, "mnnvl_multimem"),
(64 * 1024 * 1024, 1, True, False, "mnnvl_multimem"),
(64 * 1024 * 1024 + 128, 1, True, False, None),
(32 * 1024 * 1024, 0, True, False, None),
(32 * 1024 * 1024, 0, False, False, "mnnvl_multimem"),
(8 * 1024 * 1024, 1, True, True, "mnnvl_lamport"),
(32 * 1024 * 1024, 1, True, True, None),
],
)
@pytest.mark.parametrize("world_size", [2, 4, 8])
def test_mnnvl_reduce_scatter_backend_gate(
monkeypatch,
world_size,
message_bytes,
multimem_ptr,
multimem_initialized,
batch_invariant,
expected,
):
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(car.envs, "VLLM_BATCH_INVARIANT", batch_invariant)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator._ptr = 0
communicator.world_size = world_size
communicator.mnnvl_only = False
communicator.fully_connected = True
communicator.mnnvl_multicast_ptr = 1
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = multimem_initialized
communicator.mnnvl_multimem_rs_multicast_ptr = multimem_ptr
communicator.max_mnnvl_reduce_scatter_size = 16 * 1024 * 1024
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
communicator.max_reduce_scatter_size = 16 * 1024 * 1024
inp = torch.empty(
(world_size, message_bytes // torch.bfloat16.itemsize // world_size),
dtype=torch.bfloat16,
)
assert inp.nbytes == message_bytes
assert communicator._select_reduce_scatter_backend(inp) == expected
assert communicator.should_custom_reduce_scatter(inp) is (expected is not None)
assert communicator.should_mnnvl_multimem_reduce_scatter(inp) is (
expected == "mnnvl_multimem"
)
def test_mnnvl_multimem_reduce_scatter_skips_rendezvous_after_peer_alloc_failure(
monkeypatch,
):
events = []
class FakeSymmMem:
@staticmethod
def empty(*_args, **_kwargs):
events.append("empty")
return torch.empty(1, dtype=torch.uint8)
@staticmethod
def rendezvous(*_args, **_kwargs):
events.append("rendezvous")
return None
def report_peer_allocation_failure(group_value, **_kwargs):
events.append("all_reduce")
assert group_value.item() == 1
group_value.zero_()
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
monkeypatch.setattr(car.dist, "all_reduce", report_peer_allocation_failure)
warnings = []
monkeypatch.setattr(
car.logger,
"warning_once",
lambda message, *_args, **_kwargs: warnings.append(message),
)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = True
communicator._ptr = 0
communicator.group = object()
communicator.device = torch.device("cuda:0")
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = False
communicator.mnnvl_multimem_rs_buffer = None
communicator.mnnvl_multimem_rs_multicast_ptr = 0
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
assert events == ["empty", "all_reduce"]
assert communicator.mnnvl_multimem_rs_initialized
assert communicator.mnnvl_multimem_rs_buffer is None
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
assert warnings == [
"MNNVL multimem reduce-scatter symmetric-memory allocation "
"failed on at least one rank; falling back to NCCL."
]
def test_mnnvl_multimem_reduce_scatter_warns_on_rendezvous_failure(monkeypatch):
events = []
class FakeSymmMem:
@staticmethod
def empty(*_args, **_kwargs):
events.append("empty")
return torch.empty(1, dtype=torch.uint8)
@staticmethod
def rendezvous(*_args, **_kwargs):
events.append("rendezvous")
raise RuntimeError("rendezvous failed")
def preserve_local_result(_group_value, **_kwargs):
events.append("all_reduce")
warnings = []
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
monkeypatch.setattr(
car.logger,
"warning_once",
lambda message, *_args, **_kwargs: warnings.append(message),
)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = True
communicator._ptr = 0
communicator.group = type("Group", (), {"group_name": "test"})()
communicator.device = torch.device("cuda:0")
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = False
communicator.mnnvl_multimem_rs_buffer = None
communicator.mnnvl_multimem_rs_multicast_ptr = 0
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
assert events == ["empty", "all_reduce", "rendezvous", "all_reduce"]
assert communicator.mnnvl_multimem_rs_initialized
assert communicator.mnnvl_multimem_rs_buffer is None
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
assert warnings == [
"MNNVL multimem reduce-scatter symmetric-memory rendezvous "
"failed on at least one rank; falling back to NCCL."
]
def test_mnnvl_multimem_reduce_scatter_initializes_signals(monkeypatch):
events = []
buffers = []
class FakeHandle:
multicast_ptr = 0x3000
class FakeSymmMem:
@staticmethod
def empty(size, **_kwargs):
events.append(("empty", size))
buffer = torch.ones(size, dtype=torch.uint8)
buffers.append(buffer)
return buffer
@staticmethod
def rendezvous(*_args, **_kwargs):
events.append(("rendezvous", None))
return FakeHandle()
def preserve_local_result(*_args, **_kwargs):
events.append(("all_reduce", None))
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
monkeypatch.setattr(
car.torch.accelerator,
"synchronize",
lambda: events.append(("synchronize", None)),
)
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = True
communicator._ptr = 0
communicator.group = type("Group", (), {"group_name": "test"})()
communicator.device = torch.device("cpu")
communicator.max_mnnvl_multimem_reduce_scatter_size = 129
communicator.mnnvl_multimem_rs_supported = True
communicator.mnnvl_multimem_rs_initialized = False
communicator.mnnvl_multimem_rs_buffer = None
communicator.mnnvl_multimem_rs_multicast_ptr = 0
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
assert events == [
("empty", 257),
("all_reduce", None),
("rendezvous", None),
("synchronize", None),
("all_reduce", None),
]
assert torch.all(buffers[0][:128] == 0)
assert torch.all(buffers[0][128:] == 1)
assert communicator.mnnvl_multimem_rs_buffer_size == 129
assert communicator.mnnvl_multimem_rs_local_ptr == buffers[0].data_ptr() + 128
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0x3080
@ray.remote(num_gpus=1, max_calls=1)
def graph_allreduce(
monkeypatch: pytest.MonkeyPatch,