[Bugfix][Distributed] Gate cross-node MNNVL custom all-reduce by group capability (#53253)

Signed-off-by: JulianZJN <148066842+JulianZJN@users.noreply.github.com>
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
Julian
2026-08-29 08:49:24 +08:00
committed by GitHub
co-authored by Codex Summer Yang
parent 026d5af7f4
commit 99013d77d3
2 changed files with 119 additions and 0 deletions
@@ -9,6 +9,7 @@ import torch
import torch.distributed as dist
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
from vllm.distributed.device_communicators import custom_all_reduce as car
from vllm.distributed.parallel_state import get_tp_group, graph_capture
from ..utils import (
@@ -23,6 +24,67 @@ for i, v in enumerate(test_sizes):
test_sizes[i] -= v % 8
@pytest.mark.parametrize(
("major", "local_multicast", "expected"),
[
(8, True, False),
(9, True, False),
(10, False, False),
(10, True, True),
],
)
def test_cross_node_mnnvl_gate_checks_generation_and_multicast(
monkeypatch,
major,
local_multicast,
expected,
):
def has_device_capability(capability, device_id):
assert capability == 100
assert device_id == 3
return major >= 10
monkeypatch.setattr(
car.current_platform,
"has_device_capability",
has_device_capability,
)
monkeypatch.setattr(
car,
"_has_local_multicast_support",
lambda _device: local_multicast,
)
monkeypatch.setattr(car.dist, "all_reduce", lambda *_args, **_kwargs: None)
assert car._group_can_attempt_mnnvl(object(), torch.device("cuda:3")) is expected
def test_cross_node_mnnvl_gate_requires_support_on_every_rank(monkeypatch):
monkeypatch.setattr(
car.current_platform,
"has_device_capability",
lambda *_args: True,
)
monkeypatch.setattr(
car,
"_has_local_multicast_support",
lambda _device: True,
)
def report_unsupported_peer(support, **_kwargs):
support.zero_()
monkeypatch.setattr(car.dist, "all_reduce", report_unsupported_peer)
assert not car._group_can_attempt_mnnvl(object(), torch.device("cuda:0"))
def test_local_multicast_support_rejects_non_cuda(monkeypatch):
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: False)
assert not car._has_local_multicast_support(torch.device("cuda:0"))
@ray.remote(num_gpus=1, max_calls=1)
def graph_allreduce(
monkeypatch: pytest.MonkeyPatch,