[Perf][Distributed] Add low-SM multimem reduce-scatter for SM100/SM103 (#55072)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
co-authored by
Summer Yang
parent
8b365ff949
commit
442bd00d9f
@@ -23,6 +23,7 @@ import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import (
|
||||
@@ -53,6 +54,11 @@ def _supports_multimem():
|
||||
)
|
||||
|
||||
|
||||
def _supports_sm100_or_sm103():
|
||||
capability = current_platform.get_device_capability()
|
||||
return current_platform.is_cuda() and capability in ((10, 0), (10, 3))
|
||||
|
||||
|
||||
def _payload(value, device):
|
||||
return torch.full((_ROWS, _HIDDEN), value, dtype=torch.bfloat16, device=device)
|
||||
|
||||
@@ -324,3 +330,65 @@ def test_mnnvl_lamport_stage_cleanup(monkeypatch: pytest.MonkeyPatch):
|
||||
if torch.accelerator.device_count() < 2:
|
||||
pytest.skip("Need at least two GPUs to run the test.")
|
||||
multi_process_parallel(monkeypatch, 2, 1, _run_stage_cleanup_test)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def _run_multimem_reduce_scatter_test(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
|
||||
fa = get_tp_group().device_communicator.ca_comm
|
||||
assert fa is not None and not fa.disabled
|
||||
assert fa.mnnvl_multimem_rs_supported
|
||||
assert not fa.mnnvl_multimem_rs_initialized
|
||||
assert not fa.mnnvl_multimem_rs_multicast_ptr
|
||||
|
||||
message_bytes = {
|
||||
"mnnvl_lamport": 1 * 1024 * 1024,
|
||||
"mnnvl_multimem": 20 * 1024 * 1024,
|
||||
"nccl": 64 * 1024 * 1024 + 128,
|
||||
}
|
||||
for _ in range(2):
|
||||
for backend, size_bytes in message_bytes.items():
|
||||
shard_elements = size_bytes // torch.bfloat16.itemsize // tp_size
|
||||
inp = torch.full(
|
||||
(tp_size, shard_elements),
|
||||
rank + 1,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
selected = fa._select_reduce_scatter_backend(inp)
|
||||
if backend == "nccl":
|
||||
assert selected is None
|
||||
out = sp_reduce_scatter(inp)
|
||||
else:
|
||||
assert selected == backend
|
||||
out = fa.custom_reduce_scatter(inp)
|
||||
assert out is not None
|
||||
if backend == "mnnvl_multimem":
|
||||
assert fa.mnnvl_multimem_rs_initialized
|
||||
assert fa.mnnvl_multimem_rs_multicast_ptr
|
||||
expected = tp_size * (tp_size + 1) // 2
|
||||
torch.testing.assert_close(out, torch.full_like(out, expected))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _supports_sm100_or_sm103(),
|
||||
reason="The low-SM MNNVL reduce-scatter path requires SM100 or SM103.",
|
||||
)
|
||||
@pytest.mark.parametrize("tp_size", [2, 4, 8])
|
||||
def test_mnnvl_multimem_reduce_scatter(monkeypatch: pytest.MonkeyPatch, tp_size: int):
|
||||
if torch.accelerator.device_count() < tp_size:
|
||||
pytest.skip(f"Need at least {tp_size} GPUs to run the test.")
|
||||
multi_process_parallel(monkeypatch, tp_size, 1, _run_multimem_reduce_scatter_test)
|
||||
|
||||
@@ -151,9 +151,9 @@ def test_custom_allreduce_filters_dtype(
|
||||
) -> None:
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator._ptr = 0
|
||||
communicator.world_size = 2
|
||||
communicator.max_size = 1024
|
||||
communicator._ptr = 0
|
||||
|
||||
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
|
||||
|
||||
@@ -219,6 +219,268 @@ def test_local_multicast_support_rejects_non_cuda(monkeypatch):
|
||||
assert not car._has_local_multicast_support(torch.device("cuda:0"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("world_size", "device_capability", "local_multicast", "expected"),
|
||||
[
|
||||
(2, (10, 0), True, True),
|
||||
(4, (10, 3), True, True),
|
||||
(8, (10, 0), True, True),
|
||||
(8, (10, 3), True, True),
|
||||
(6, (10, 3), True, False),
|
||||
(8, (10, 1), True, False),
|
||||
(8, (9, 0), True, False),
|
||||
(8, (10, 3), False, False),
|
||||
],
|
||||
)
|
||||
def test_mnnvl_multimem_reduce_scatter_platform_gate(
|
||||
monkeypatch,
|
||||
world_size,
|
||||
device_capability,
|
||||
local_multicast,
|
||||
expected,
|
||||
):
|
||||
def is_device_capability(capability, device_id):
|
||||
assert capability in ((10, 0), (10, 3))
|
||||
assert device_id == 3
|
||||
return device_capability == capability
|
||||
|
||||
monkeypatch.setattr(
|
||||
car.current_platform,
|
||||
"is_device_capability",
|
||||
is_device_capability,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
car,
|
||||
"_has_local_multicast_support",
|
||||
lambda _device: local_multicast,
|
||||
)
|
||||
|
||||
supported = car._supports_mnnvl_multimem_reduce_scatter(
|
||||
torch.device("cuda:3"), world_size
|
||||
)
|
||||
assert supported is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"message_bytes",
|
||||
"multimem_ptr",
|
||||
"multimem_initialized",
|
||||
"batch_invariant",
|
||||
"expected",
|
||||
),
|
||||
[
|
||||
(16 * 1024 * 1024, 1, True, False, "mnnvl_lamport"),
|
||||
(16 * 1024 * 1024 + 128, 1, True, False, "mnnvl_multimem"),
|
||||
(64 * 1024 * 1024, 1, True, False, "mnnvl_multimem"),
|
||||
(64 * 1024 * 1024 + 128, 1, True, False, None),
|
||||
(32 * 1024 * 1024, 0, True, False, None),
|
||||
(32 * 1024 * 1024, 0, False, False, "mnnvl_multimem"),
|
||||
(8 * 1024 * 1024, 1, True, True, "mnnvl_lamport"),
|
||||
(32 * 1024 * 1024, 1, True, True, None),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("world_size", [2, 4, 8])
|
||||
def test_mnnvl_reduce_scatter_backend_gate(
|
||||
monkeypatch,
|
||||
world_size,
|
||||
message_bytes,
|
||||
multimem_ptr,
|
||||
multimem_initialized,
|
||||
batch_invariant,
|
||||
expected,
|
||||
):
|
||||
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(car.envs, "VLLM_BATCH_INVARIANT", batch_invariant)
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator._ptr = 0
|
||||
communicator.world_size = world_size
|
||||
communicator.mnnvl_only = False
|
||||
communicator.fully_connected = True
|
||||
communicator.mnnvl_multicast_ptr = 1
|
||||
communicator.mnnvl_multimem_rs_supported = True
|
||||
communicator.mnnvl_multimem_rs_initialized = multimem_initialized
|
||||
communicator.mnnvl_multimem_rs_multicast_ptr = multimem_ptr
|
||||
communicator.max_mnnvl_reduce_scatter_size = 16 * 1024 * 1024
|
||||
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
|
||||
communicator.max_reduce_scatter_size = 16 * 1024 * 1024
|
||||
inp = torch.empty(
|
||||
(world_size, message_bytes // torch.bfloat16.itemsize // world_size),
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
assert inp.nbytes == message_bytes
|
||||
assert communicator._select_reduce_scatter_backend(inp) == expected
|
||||
assert communicator.should_custom_reduce_scatter(inp) is (expected is not None)
|
||||
assert communicator.should_mnnvl_multimem_reduce_scatter(inp) is (
|
||||
expected == "mnnvl_multimem"
|
||||
)
|
||||
|
||||
|
||||
def test_mnnvl_multimem_reduce_scatter_skips_rendezvous_after_peer_alloc_failure(
|
||||
monkeypatch,
|
||||
):
|
||||
events = []
|
||||
|
||||
class FakeSymmMem:
|
||||
@staticmethod
|
||||
def empty(*_args, **_kwargs):
|
||||
events.append("empty")
|
||||
return torch.empty(1, dtype=torch.uint8)
|
||||
|
||||
@staticmethod
|
||||
def rendezvous(*_args, **_kwargs):
|
||||
events.append("rendezvous")
|
||||
return None
|
||||
|
||||
def report_peer_allocation_failure(group_value, **_kwargs):
|
||||
events.append("all_reduce")
|
||||
assert group_value.item() == 1
|
||||
group_value.zero_()
|
||||
|
||||
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
|
||||
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
|
||||
monkeypatch.setattr(car.dist, "all_reduce", report_peer_allocation_failure)
|
||||
warnings = []
|
||||
monkeypatch.setattr(
|
||||
car.logger,
|
||||
"warning_once",
|
||||
lambda message, *_args, **_kwargs: warnings.append(message),
|
||||
)
|
||||
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = True
|
||||
communicator._ptr = 0
|
||||
communicator.group = object()
|
||||
communicator.device = torch.device("cuda:0")
|
||||
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
|
||||
communicator.mnnvl_multimem_rs_supported = True
|
||||
communicator.mnnvl_multimem_rs_initialized = False
|
||||
communicator.mnnvl_multimem_rs_buffer = None
|
||||
communicator.mnnvl_multimem_rs_multicast_ptr = 0
|
||||
|
||||
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
|
||||
|
||||
assert events == ["empty", "all_reduce"]
|
||||
assert communicator.mnnvl_multimem_rs_initialized
|
||||
assert communicator.mnnvl_multimem_rs_buffer is None
|
||||
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
|
||||
assert warnings == [
|
||||
"MNNVL multimem reduce-scatter symmetric-memory allocation "
|
||||
"failed on at least one rank; falling back to NCCL."
|
||||
]
|
||||
|
||||
|
||||
def test_mnnvl_multimem_reduce_scatter_warns_on_rendezvous_failure(monkeypatch):
|
||||
events = []
|
||||
|
||||
class FakeSymmMem:
|
||||
@staticmethod
|
||||
def empty(*_args, **_kwargs):
|
||||
events.append("empty")
|
||||
return torch.empty(1, dtype=torch.uint8)
|
||||
|
||||
@staticmethod
|
||||
def rendezvous(*_args, **_kwargs):
|
||||
events.append("rendezvous")
|
||||
raise RuntimeError("rendezvous failed")
|
||||
|
||||
def preserve_local_result(_group_value, **_kwargs):
|
||||
events.append("all_reduce")
|
||||
|
||||
warnings = []
|
||||
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
|
||||
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
|
||||
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
|
||||
monkeypatch.setattr(
|
||||
car.logger,
|
||||
"warning_once",
|
||||
lambda message, *_args, **_kwargs: warnings.append(message),
|
||||
)
|
||||
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = True
|
||||
communicator._ptr = 0
|
||||
communicator.group = type("Group", (), {"group_name": "test"})()
|
||||
communicator.device = torch.device("cuda:0")
|
||||
communicator.max_mnnvl_multimem_reduce_scatter_size = 64 * 1024 * 1024
|
||||
communicator.mnnvl_multimem_rs_supported = True
|
||||
communicator.mnnvl_multimem_rs_initialized = False
|
||||
communicator.mnnvl_multimem_rs_buffer = None
|
||||
communicator.mnnvl_multimem_rs_multicast_ptr = 0
|
||||
|
||||
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
|
||||
|
||||
assert events == ["empty", "all_reduce", "rendezvous", "all_reduce"]
|
||||
assert communicator.mnnvl_multimem_rs_initialized
|
||||
assert communicator.mnnvl_multimem_rs_buffer is None
|
||||
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0
|
||||
assert warnings == [
|
||||
"MNNVL multimem reduce-scatter symmetric-memory rendezvous "
|
||||
"failed on at least one rank; falling back to NCCL."
|
||||
]
|
||||
|
||||
|
||||
def test_mnnvl_multimem_reduce_scatter_initializes_signals(monkeypatch):
|
||||
events = []
|
||||
buffers = []
|
||||
|
||||
class FakeHandle:
|
||||
multicast_ptr = 0x3000
|
||||
|
||||
class FakeSymmMem:
|
||||
@staticmethod
|
||||
def empty(size, **_kwargs):
|
||||
events.append(("empty", size))
|
||||
buffer = torch.ones(size, dtype=torch.uint8)
|
||||
buffers.append(buffer)
|
||||
return buffer
|
||||
|
||||
@staticmethod
|
||||
def rendezvous(*_args, **_kwargs):
|
||||
events.append(("rendezvous", None))
|
||||
return FakeHandle()
|
||||
|
||||
def preserve_local_result(*_args, **_kwargs):
|
||||
events.append(("all_reduce", None))
|
||||
|
||||
monkeypatch.setattr(car, "torch_symm_mem", FakeSymmMem)
|
||||
monkeypatch.setattr(car.ops, "meta_size", lambda: 128)
|
||||
monkeypatch.setattr(
|
||||
car.torch.accelerator,
|
||||
"synchronize",
|
||||
lambda: events.append(("synchronize", None)),
|
||||
)
|
||||
monkeypatch.setattr(car.dist, "all_reduce", preserve_local_result)
|
||||
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = True
|
||||
communicator._ptr = 0
|
||||
communicator.group = type("Group", (), {"group_name": "test"})()
|
||||
communicator.device = torch.device("cpu")
|
||||
communicator.max_mnnvl_multimem_reduce_scatter_size = 129
|
||||
communicator.mnnvl_multimem_rs_supported = True
|
||||
communicator.mnnvl_multimem_rs_initialized = False
|
||||
communicator.mnnvl_multimem_rs_buffer = None
|
||||
communicator.mnnvl_multimem_rs_multicast_ptr = 0
|
||||
|
||||
communicator._init_mnnvl_multimem_reduce_scatter_buffer()
|
||||
|
||||
assert events == [
|
||||
("empty", 257),
|
||||
("all_reduce", None),
|
||||
("rendezvous", None),
|
||||
("synchronize", None),
|
||||
("all_reduce", None),
|
||||
]
|
||||
assert torch.all(buffers[0][:128] == 0)
|
||||
assert torch.all(buffers[0][128:] == 1)
|
||||
assert communicator.mnnvl_multimem_rs_buffer_size == 129
|
||||
assert communicator.mnnvl_multimem_rs_local_ptr == buffers[0].data_ptr() + 128
|
||||
assert communicator.mnnvl_multimem_rs_multicast_ptr == 0x3080
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def graph_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
||||
Reference in New Issue
Block a user