[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
-95
View File
@@ -1,95 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import atexit
import os
import random
import pytest
import torch
import torch.multiprocessing as mp
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed.parallel_state import (
init_distributed_environment,
)
from vllm.platforms import current_platform
from vllm.utils.system_utils import update_environment_variables
mp.set_start_method("spawn", force=True)
def _distributed_worker_wrapper(fn, env, world_size, args, rank, skip_queue):
try:
fn(env, world_size, *args)
except BaseException as exc:
if isinstance(exc, pytest.skip.Exception):
skip_queue.put((rank, str(exc)))
return
raise
def distributed_run(fn, world_size, *args):
number_of_processes = world_size
processes: list[mp.Process] = []
skip_queue: mp.SimpleQueue = mp.SimpleQueue()
for i in range(number_of_processes):
env: dict[str, str] = {}
env["RANK"] = str(i)
env["LOCAL_RANK"] = str(i)
env["WORLD_SIZE"] = str(number_of_processes)
env["LOCAL_WORLD_SIZE"] = str(number_of_processes)
env["MASTER_ADDR"] = "localhost"
env["MASTER_PORT"] = "12345"
p = mp.Process(
target=_distributed_worker_wrapper,
args=(fn, env, world_size, args, i, skip_queue),
)
processes.append(p)
p.start()
for p in processes:
p.join()
skipped: list[tuple[int, str]] = []
while not skip_queue.empty():
rank, reason = skip_queue.get()
skipped.append((rank, reason))
if len(skipped) == number_of_processes:
reason = skipped[0][1]
pytest.skip(reason)
if 0 < len(skipped) < number_of_processes:
skipped_ranks = sorted(rank for rank, _ in skipped)
raise AssertionError(
"Distributed test had partial skips; expected either all ranks "
f"to skip or none. Skipped ranks: {skipped_ranks}, "
f"total ranks: {number_of_processes}"
)
for p in processes:
assert p.exitcode == 0
def set_env_vars_and_device(env: dict[str, str]) -> None:
update_environment_variables(env)
local_rank = os.environ["LOCAL_RANK"]
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{local_rank}")
torch.accelerator.set_device_index(device)
backend = "xccl" if current_platform.is_xpu() else "nccl"
# Create a minimal vllm config for init_distributed_environment
vllm_config = VllmConfig()
with set_current_vllm_config(vllm_config):
init_distributed_environment(backend=backend)
atexit.register(_destroy_process_group_if_initialized)
# Ensure each worker process has the same random seed
random.seed(42)
torch.manual_seed(42)
def _destroy_process_group_if_initialized() -> None:
if torch.distributed.is_available() and torch.distributed.is_initialized():
torch.distributed.destroy_process_group()
@@ -1,95 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
import types
from types import SimpleNamespace
import pytest
import torch
import vllm._aiter_ops as aiter_ops
import vllm.distributed as vllm_distributed
import vllm.kernels # noqa: F401
from vllm.platforms import current_platform
def test_rocm_aiter_gemma_rmsnorm_falls_back_after_allreduce(
monkeypatch: pytest.MonkeyPatch,
):
class FakeAiterCA:
world_size = 8
fully_connected = True
def custom_fused_ar_rms(self, *args, **kwargs):
return None
class FakeAiterAllReduce:
aiter_ca = FakeAiterCA()
def use_1stage_fused_ar_rms(self, inp):
return False
class FakeQuickReduce:
disabled = False
def should_quick_allreduce(self, inp):
raise AssertionError("Gemma norm must not use fused QR+RMSNorm")
device_comm = SimpleNamespace(qr_comm=FakeQuickReduce())
monkeypatch.setattr(
vllm_distributed,
"get_tp_group",
lambda: SimpleNamespace(device_communicator=device_comm),
)
monkeypatch.setattr(
aiter_ops.rocm_aiter_ops,
"get_aiter_allreduce",
lambda: FakeAiterAllReduce(),
)
monkeypatch.setattr(
vllm_distributed,
"tensor_model_parallel_all_reduce",
lambda inp: inp + 2,
)
calls = []
fake_aiter = types.SimpleNamespace()
def fake_rmsnorm2d_fwd_with_add(
out,
inp,
residual,
residual_out,
weight,
epsilon,
*,
gemma_norm,
):
calls.append((inp, weight, epsilon, gemma_norm))
residual_out.copy_(inp + residual)
out.copy_(residual_out)
fake_aiter.rmsnorm2d_fwd_with_add = fake_rmsnorm2d_fwd_with_add
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
# On the platform's device, not the CPU. The fallback leaves Python through
# `torch.ops.vllm_aiter.fused_add_rms_norm`, which is registered for the
# platform's dispatch key alone, so CPU tensors have nothing to dispatch to
# on a GPU build. The kernel behind it is still the fake above.
device = current_platform.device_type
inp = torch.randn(8192, 16, dtype=torch.bfloat16, device=device)
residual = torch.randn_like(inp)
weight = torch.randn(16, dtype=torch.bfloat16, device=device)
out, residual_out = aiter_ops._rocm_aiter_fused_allreduce_rmsnorm_impl(
inp, residual, weight, 1e-6, gemma_norm=True
)
assert len(calls) == 1
assert calls[0][0] is not inp
assert calls[0][1] is weight
assert calls[0][2] == 1e-6
assert calls[0][3] is True
torch.testing.assert_close(residual_out, inp + 2 + residual)
torch.testing.assert_close(out, residual_out)
-785
View File
@@ -1,785 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test the communication operators.
Run `pytest tests/distributed/test_comm_ops.py`.
"""
from collections import deque
from collections.abc import Callable
from typing import Any
from unittest.mock import Mock
import pytest
import ray
import torch
from vllm.distributed import (
broadcast_tensor_dict,
get_pp_group,
tensor_model_parallel_all_gather,
tensor_model_parallel_all_reduce,
tensor_model_parallel_reduce_scatter,
)
from vllm.distributed.device_communicators import flashinfer_all_reduce
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
from vllm.distributed.parallel_state import GroupCoordinator, TensorMetadata
from vllm.v1.worker.gpu_worker import AsyncIntermediateTensors
from ..utils import (
init_test_distributed_environment,
multi_gpu_test,
multi_process_parallel,
)
@ray.remote(num_gpus=1, max_calls=1)
def all_reduce_test_worker(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
pp_size: int,
rank: int,
distributed_init_port: str,
):
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
# so that each worker can see all the GPUs
# they will be able to set the device to the correct GPU
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
num_elements = 8
all_tensors = [
torch.arange(num_elements, dtype=torch.float32, device="cuda") * (r + 1)
for r in range(tp_size)
]
expected = torch.sum(torch.stack(all_tensors, dim=0), dim=0)
t = all_tensors[rank % tp_size]
t = tensor_model_parallel_all_reduce(t)
torch.testing.assert_close(t, expected)
@ray.remote(num_gpus=1, max_calls=1)
def reduce_scatter_test_worker(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
pp_size: int,
rank: int,
distributed_init_port: str,
):
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
# so that each worker can see all the GPUs
# they will be able to set the device to the correct GPU
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
num_elements = 8
all_tensors = [
torch.arange(num_elements, dtype=torch.float32, device="cuda") * (r + 1)
for r in range(tp_size)
]
index = rank % tp_size
partition_size = num_elements // tp_size
all_reduce = torch.sum(torch.stack(all_tensors, dim=0), dim=0)
expected = all_reduce[index * partition_size : (index + 1) * partition_size]
t = all_tensors[index]
t = tensor_model_parallel_reduce_scatter(t, 0)
torch.testing.assert_close(t, expected)
@ray.remote(num_gpus=1, max_calls=1)
def all_gather_test_worker(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
pp_size: int,
rank: int,
distributed_init_port: str,
):
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
# so that each worker can see all the GPUs
# they will be able to set the device to the correct GPU
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
num_dimensions = 3
tensor_size = list(range(2, num_dimensions + 2))
total_size = 1
for s in tensor_size:
total_size *= s
for all_gather_dimension in range(num_dimensions):
all_tensors = [
torch.arange(total_size, dtype=torch.float32, device="cuda").reshape(
tensor_size
)
* (r + 1)
for r in range(tp_size)
]
expected = torch.cat(all_tensors, dim=all_gather_dimension)
t = all_tensors[rank % tp_size]
t = tensor_model_parallel_all_gather(t, all_gather_dimension)
torch.testing.assert_close(t, expected)
@ray.remote(num_gpus=1, max_calls=1)
def broadcast_tensor_dict_test_worker(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
pp_size: int,
rank: int,
distributed_init_port: str,
):
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
# so that each worker can see all the GPUs
# they will be able to set the device to the correct GPU
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
test_dict = {
# device tensor
"a": torch.arange(8, dtype=torch.float32, device="cuda"),
# CPU tensor
"b": torch.arange(16, dtype=torch.int8, device="cpu"),
"c": "test",
"d": [1, 2, 3],
"e": {"a": 1, "b": 2},
# empty tensor
"f": torch.tensor([], dtype=torch.float32, device="cuda"),
}
if (rank % tp_size) == 0:
broadcast_tensor_dict(test_dict, src=0)
else:
recv_dict = broadcast_tensor_dict(src=0)
assert len(recv_dict) == len(test_dict)
torch.testing.assert_close(recv_dict["a"], test_dict["a"])
torch.testing.assert_close(recv_dict["b"], test_dict["b"])
assert recv_dict["c"] == test_dict["c"]
assert recv_dict["d"] == test_dict["d"]
assert recv_dict["e"] == test_dict["e"]
torch.testing.assert_close(recv_dict["f"], test_dict["f"])
@ray.remote(num_gpus=1, max_calls=1)
def send_recv_tensor_dict_test_worker(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
pp_size: int,
rank: int,
distributed_init_port: str,
):
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
test_dict = {
# device tensor
"a": torch.arange(8, dtype=torch.float32, device="cuda"),
# CPU tensor
"b": torch.arange(16, dtype=torch.int8, device="cpu"),
"c": "test",
"d": [1, 2, 3],
"e": {"a": 1, "b": 2},
# empty tensor
"f": torch.tensor([], dtype=torch.float32, device="cuda"),
}
if not get_pp_group().is_first_rank:
recv_dict = get_pp_group().recv_tensor_dict()
if not get_pp_group().is_last_rank:
get_pp_group().send_tensor_dict(test_dict)
if not get_pp_group().is_first_rank:
assert len(recv_dict) == len(test_dict)
torch.testing.assert_close(recv_dict["a"], test_dict["a"])
torch.testing.assert_close(recv_dict["b"], test_dict["b"])
assert recv_dict["c"] == test_dict["c"]
assert recv_dict["d"] == test_dict["d"]
assert recv_dict["e"] == test_dict["e"]
torch.testing.assert_close(recv_dict["f"], test_dict["f"])
class _DummyWork:
def __init__(self) -> None:
self.wait_calls = 0
self.completed = False
def wait(self) -> None:
self.wait_calls += 1
self.completed = True
def is_completed(self) -> bool:
return self.completed
class _DummyAllGatherGroup:
def __init__(self, world_size: int, rank_in_group: int) -> None:
self.world_size = world_size
self.rank_in_group = rank_in_group
def all_gather(self, t: torch.Tensor, dim: int = 0) -> torch.Tensor:
# duplicate local slice across ranks.
assert dim == 0
return torch.cat([t for _ in range(self.world_size)], dim=0)
def _make_group_for_unit_test(
rank_in_group: int = 0, world_size: int = 2
) -> GroupCoordinator:
# avoid running GroupCoordinator.__init__ (it wires up real process groups).
g = GroupCoordinator.__new__(GroupCoordinator)
g.world_size = world_size
g.rank_in_group = rank_in_group
g.ranks = list(range(world_size))
g.use_cpu_custom_send_recv = False
g.device_group = None
g.cpu_group = None
g._pending_isends = deque()
return g
def test_irecv_tensor_dict_send_allgather_postprocess_binds_keys(
monkeypatch: pytest.MonkeyPatch,
) -> None:
def fake_irecv(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
t.fill_(1)
return _DummyWork()
monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True)
monkeypatch.setattr(torch.distributed, "irecv", fake_irecv)
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
# 2 tensors so we can catch late-binding bugs in postprocess closures.
metadata_list = [
("a", TensorMetadata("cpu", torch.int32, torch.Size([4]))),
("b", TensorMetadata("cpu", torch.int32, torch.Size([4]))),
]
g.recv_object = lambda src=None: metadata_list # type: ignore[method-assign]
ag = _DummyAllGatherGroup(world_size=2, rank_in_group=0)
td, handles, postprocess = g.irecv_tensor_dict(all_gather_group=ag)
assert td is not None
assert len(handles) == 2
assert len(postprocess) == 2
# before postprocess, dict holds the TP slice (shape 2).
assert td["a"].shape == torch.Size([2])
assert td["b"].shape == torch.Size([2])
# simulate worker-side "defer wait": wait + postprocess later.
for handle in handles:
handle.wait()
for fn in postprocess:
fn()
# after postprocess, dict values are reconstructed to full shape (shape 4),
# and each key should be updated independently
assert td["a"].shape == torch.Size([4])
assert td["b"].shape == torch.Size([4])
torch.testing.assert_close(td["a"], torch.ones(4, dtype=torch.int32))
torch.testing.assert_close(td["b"], torch.ones(4, dtype=torch.int32))
@pytest.mark.parametrize("aliased", [False, True])
def test_cuda_communicator_checkpoints_flashinfer_workspaces(
monkeypatch: pytest.MonkeyPatch,
aliased: bool,
) -> None:
group = object()
normal_workspace = Mock()
quant_workspace = normal_workspace if aliased else Mock()
unique_workspaces = (
[normal_workspace] if aliased else [normal_workspace, quant_workspace]
)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", normal_workspace)
monkeypatch.setattr(
flashinfer_all_reduce, "_fi_ar_quant_workspace", quant_workspace
)
monkeypatch.setattr(
flashinfer_all_reduce,
"_fi_ar_workspace_groups",
{id(workspace): group for workspace in unique_workspaces},
)
monkeypatch.setattr(
flashinfer_all_reduce, "TorchDistBackend", lambda group: group, raising=False
)
communicator = CudaCommunicator.__new__(CudaCommunicator)
communicator.cpu_group = group
communicator.fi_ar_comm = None
communicator.all2all_manager = None
communicator.checkpoint_prepare()
communicator.checkpoint_restore()
for workspace in unique_workspaces:
workspace.checkpoint_prepare.assert_called_once_with()
workspace.checkpoint_restore.assert_called_once_with(group)
def _fi_ar_communicator(
monkeypatch: pytest.MonkeyPatch, group: object, normal: Mock, quant: Mock
) -> CudaCommunicator:
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", normal)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", quant)
monkeypatch.setattr(
flashinfer_all_reduce,
"_fi_ar_workspace_groups",
{id(normal): group, id(quant): group},
)
monkeypatch.setattr(
flashinfer_all_reduce, "TorchDistBackend", lambda group: group, raising=False
)
communicator = CudaCommunicator.__new__(CudaCommunicator)
communicator.cpu_group = group
communicator.pynccl_comm = Mock()
communicator.all2all_manager = None
return communicator
def _symm_mem_workspace() -> Mock:
workspace = Mock()
workspace.checkpoint_prepare.side_effect = NotImplementedError
workspace.checkpoint_restore.side_effect = NotImplementedError
return workspace
@pytest.mark.parametrize("symm_mem", [False, True], ids=["mnnvl", "symm-mem+mnnvl"])
def test_cuda_communicator_suspend_detaches_flashinfer_workspaces(
monkeypatch: pytest.MonkeyPatch,
symm_mem: bool,
) -> None:
group = object()
workspaces = (_symm_mem_workspace() if symm_mem else Mock(), Mock())
communicator = _fi_ar_communicator(monkeypatch, group, *workspaces)
communicator.suspend()
communicator.resume()
for workspace in workspaces:
workspace.checkpoint_prepare.assert_called_once_with()
workspace.checkpoint_restore.assert_called_once_with(group)
communicator.pynccl_comm.suspend.assert_called_once_with()
communicator.pynccl_comm.resume.assert_called_once_with()
def test_cuda_communicator_checkpoint_rejects_symm_mem_workspace(
monkeypatch: pytest.MonkeyPatch,
) -> None:
communicator = _fi_ar_communicator(
monkeypatch, object(), _symm_mem_workspace(), Mock()
)
with pytest.raises(NotImplementedError):
communicator.checkpoint_prepare()
with pytest.raises(NotImplementedError):
communicator.checkpoint_restore()
def test_cuda_communicator_resume_retries_failed_restore(
monkeypatch: pytest.MonkeyPatch,
) -> None:
workspace = Mock()
workspace.checkpoint_restore.side_effect = [RuntimeError("cuMemCreate"), None]
communicator = _fi_ar_communicator(monkeypatch, object(), workspace, Mock())
communicator.suspend()
with pytest.raises(RuntimeError):
communicator.resume()
communicator.resume()
assert workspace.checkpoint_restore.call_count == 2
@pytest.mark.parametrize(
("backend", "capability", "world_size", "nodes", "expected"),
[
("mnnvl", 103, 4, 1, 80 * flashinfer_all_reduce.MiB - 1),
("mnnvl", 103, 8, 2, 64 * flashinfer_all_reduce.MiB - 1),
("mnnvl", 103, 16, 4, 8 * flashinfer_all_reduce.MiB - 1),
("mnnvl", 103, 2, 1, None),
("mnnvl", 103, 12, 3, None),
("mnnvl", 103, 8, 1, 64 * flashinfer_all_reduce.MiB - 1),
("trtllm", 103, 8, 2, None),
("mnnvl", 90, 8, 2, None),
],
)
def test_flashinfer_standalone_size_tuning(
monkeypatch: pytest.MonkeyPatch,
backend: str,
capability: int,
world_size: int,
nodes: int,
expected: int | None,
) -> None:
monkeypatch.setattr(
flashinfer_all_reduce,
"current_platform",
Mock(get_device_capability=lambda: Mock(to_int=lambda: capability)),
)
monkeypatch.setattr(flashinfer_all_reduce, "_node_count", lambda _: nodes)
assert (
flashinfer_all_reduce._get_tuned_standalone_max_size(
world_size, backend, Mock()
)
== expected
)
@pytest.mark.parametrize(("enabled", "expected"), [(True, 4681), (False, 128)])
def test_flashinfer_standalone_workspace_size(
monkeypatch: pytest.MonkeyPatch, enabled: bool, expected: int
) -> None:
create_workspace = Mock(return_value=Mock(backend="mnnvl"))
monkeypatch.setattr(
flashinfer_all_reduce.envs, "VLLM_ALLREDUCE_USE_FLASHINFER", enabled
)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", None)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", None)
monkeypatch.setattr(
flashinfer_all_reduce,
"_resolve_fi_ar_backend",
Mock(return_value=("mnnvl", False)),
)
monkeypatch.setattr(
flashinfer_all_reduce,
"_get_tuned_standalone_max_size",
Mock(return_value=64 * flashinfer_all_reduce.MiB - 1),
)
monkeypatch.setattr(flashinfer_all_reduce, "_create_workspace", create_workspace)
flashinfer_all_reduce.get_fi_ar_workspace(8, 0, 128, 7168, torch.bfloat16, Mock())
assert create_workspace.call_args.args[3] == expected
@pytest.mark.parametrize(
("node_count", "is_blackwell", "expected"),
[
(1, False, ("mnnvl", True)),
(1, True, ("mnnvl", True)),
(2, False, (None, False)),
(2, True, ("mnnvl", False)),
],
)
def test_flashinfer_auto_backend_uses_group_topology(
monkeypatch: pytest.MonkeyPatch,
node_count: int,
is_blackwell: bool,
expected: tuple[str | None, bool],
) -> None:
monkeypatch.setattr(
flashinfer_all_reduce.envs, "VLLM_FLASHINFER_ALLREDUCE_BACKEND", "auto"
)
monkeypatch.setattr(flashinfer_all_reduce, "_node_count", lambda _: node_count)
monkeypatch.setattr(
flashinfer_all_reduce.current_platform,
"has_device_capability",
lambda capability: is_blackwell,
)
assert flashinfer_all_reduce._resolve_fi_ar_backend(Mock()) == expected
def test_flashinfer_workspace_failure_is_not_retried(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Creation is collective; retrying it per all-reduce desyncs ranks."""
create_workspace = Mock(return_value=None)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", None)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", None)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace_failed", False)
monkeypatch.setattr(
flashinfer_all_reduce,
"_resolve_fi_ar_backend",
Mock(return_value=("mnnvl", True)),
)
monkeypatch.setattr(
flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None)
)
monkeypatch.setattr(flashinfer_all_reduce, "_create_workspace", create_workspace)
args = (2, 0, 128, 4096, torch.bfloat16, Mock())
assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None
assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None
# One mnnvl attempt plus one trtllm fallback, never repeated.
assert create_workspace.call_count == 2
flashinfer_all_reduce.destroy_fi_ar_workspace()
flashinfer_all_reduce.get_fi_ar_workspace(*args)
assert create_workspace.call_count == 4
def test_flashinfer_all_reduce_precedes_nccl(monkeypatch: pytest.MonkeyPatch) -> None:
output = torch.empty(2)
fi_ar_comm = Mock(disabled=False)
fi_ar_comm.should_use_fi_ar.return_value = True
fi_ar_comm.all_reduce.return_value = output
communicator = CudaCommunicator.__new__(CudaCommunicator)
communicator.fi_ar_comm = fi_ar_comm
communicator.fi_pcie_ipc_ar_comm = None
communicator.pynccl_comm = Mock(world_size=8)
communicator.qr_comm = None
nccl_selector = Mock(return_value=True)
monkeypatch.setattr(
"vllm.distributed.device_communicators.cuda_communicator."
"should_nccl_symm_mem_allreduce",
nccl_selector,
)
assert communicator.all_reduce(torch.empty(1)) is output
nccl_selector.assert_not_called()
def test_aiter_all_gather_precedes_pynccl(monkeypatch: pytest.MonkeyPatch) -> None:
"""Ensure can use aiter all_gather if available even if pynccl is not available."""
output = torch.empty(2)
aiter_comm = Mock()
aiter_comm.should_custom_ag.return_value = True
aiter_comm.custom_all_gather.return_value = output
communicator = CudaCommunicator.__new__(CudaCommunicator)
communicator.world_size = 2
communicator.aiter_ar_comm = aiter_comm
communicator.pynccl_comm = Mock(disabled=True)
monkeypatch.setattr(communicator, "_can_use_aiter_ag_rs", Mock(return_value=True))
assert communicator.all_gatherv(torch.empty(1)) is output
def test_isend_object_posts_size_then_object_and_releases_on_wait(
monkeypatch: pytest.MonkeyPatch,
) -> None:
posted: list[tuple[torch.Tensor, _DummyWork]] = []
def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
w = _DummyWork()
posted.append((t, w))
return w
monkeypatch.setattr(torch.distributed, "isend", fake_isend)
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
handle = g.isend_object({"k": [1, 2, 3]}, dst=1)
# two sends, in size-then-object order (preserves gloo FIFO).
assert len(posted) == 2
assert posted[0][0].dtype == torch.long
assert posted[0][0].shape == torch.Size([1])
assert posted[0][0].item() == posted[1][0].numel()
assert posted[1][0].dtype == torch.uint8
# retain holds both source tensors until wait.
assert handle._retained == (posted[0][0], posted[1][0])
handle.wait()
# both underlying works drained, retain dropped.
assert all(w.wait_calls == 1 for _, w in posted)
assert handle._retained == ()
# wait is idempotent: gloo Work.wait() is single-shot for p2p sends (a
# second wait blocks forever), and both the lazy FIFO reap in
# ``_reap_completed_isends`` and any explicit isend caller may wait the
# same metadata handle.
handle.wait()
assert all(w.wait_calls == 1 for _, w in posted)
assert handle._retained == ()
def test_isend_tensor_dict_includes_metadata_handle(
monkeypatch: pytest.MonkeyPatch,
) -> None:
posted: list[torch.Tensor] = []
def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
posted.append(t)
return _DummyWork()
monkeypatch.setattr(torch.distributed, "isend", fake_isend)
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
td = {"a": torch.arange(4, dtype=torch.float32, device="cpu")}
handles = g.isend_tensor_dict(td, dst=1)
# size + object (metadata) + one tensor send.
assert len(posted) == 3
assert posted[0].dtype == torch.long
assert posted[1].dtype == torch.uint8
assert posted[2].dtype == torch.float32
# composite metadata handle + one tensor handle.
assert len(handles) == 2
for handle in handles:
handle.wait()
assert handles[0]._retained == ()
def test_isend_tensor_dict_self_retains_for_fire_and_forget(
monkeypatch: pytest.MonkeyPatch,
) -> None:
works: list[_DummyWork] = []
def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
w = _DummyWork()
works.append(w)
return w
monkeypatch.setattr(torch.distributed, "isend", fake_isend)
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
td = {"a": torch.arange(4, dtype=torch.float32, device="cpu")}
# fire-and-forget: the returned handles are dropped by the caller, so the
# group must self-retain the handles and the source tensor.
g.isend_tensor_dict(td, dst=1)
assert len(g._pending_isends) == 1
handles0, tensors0 = g._pending_isends[0]
assert len(handles0) == 2 # metadata composite + one tensor send
assert tensors0 == [td["a"]]
# the first send is still in flight, so the next call's reap keeps it.
g.isend_tensor_dict(td, dst=1)
assert len(g._pending_isends) == 2
# metadata-only send (no tensors) is dropped best-effort on the next
# reap, since it has no reliable completion signal.
g.isend_tensor_dict({"meta": "no-tensors"}, dst=1)
assert len(g._pending_isends) == 3
# complete every posted work; the next call reaps all three old entries
# (the metadata-only one unconditionally) and keeps only its own.
for w in works:
w.completed = True
g.isend_tensor_dict(td, dst=1)
assert len(g._pending_isends) == 1
# reaped entries had their metadata handles waited as a backstop.
assert handles0[0]._retained == ()
def test_send_tensor_dict_sync_path_does_not_self_retain(
monkeypatch: pytest.MonkeyPatch,
) -> None:
def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
return _DummyWork()
monkeypatch.setattr(torch.distributed, "isend", fake_isend)
monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True)
monkeypatch.setattr(GroupCoordinator, "send_object", lambda self, obj, dst: None)
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
td = {"a": torch.arange(4, dtype=torch.float32, device="cpu")}
g.send_tensor_dict(td, dst=1)
# the sync path waited every handle, so no retention entry may leak.
assert len(g._pending_isends) == 0
def test_async_intermediate_tensors_lazy_wait() -> None:
work = _DummyWork()
post_calls = {"n": 0}
def post() -> None:
post_calls["n"] += 1
it = AsyncIntermediateTensors(
{"x": torch.tensor([1])},
comm_handles=[work],
comm_postprocess=[post],
)
# accessing non-tensor attributes should not trigger wait.
assert it._comm_handles is not None
assert work.wait_calls == 0
assert post_calls["n"] == 0
# first access of `.tensors` triggers wait + postprocess.
_ = it.tensors
assert work.wait_calls == 1
assert post_calls["n"] == 1
# subsequent access should not re-wait.
_ = it.tensors
assert work.wait_calls == 1
assert post_calls["n"] == 1
@ray.remote(num_gpus=1, max_calls=1)
def send_recv_test_worker(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
pp_size: int,
rank: int,
distributed_init_port: str,
):
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
size = 64
test_tensor = torch.arange(64, dtype=torch.float32, device="cuda")
if not get_pp_group().is_first_rank:
recv_tensor = get_pp_group().recv(size, dtype=torch.float32)
if not get_pp_group().is_last_rank:
get_pp_group().send(test_tensor)
if not get_pp_group().is_first_rank:
torch.testing.assert_close(test_tensor, recv_tensor)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("tp_size", [2])
@pytest.mark.parametrize(
"test_target",
[all_reduce_test_worker, all_gather_test_worker, broadcast_tensor_dict_test_worker],
)
def test_multi_process_tensor_parallel(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
test_target: Callable[..., Any],
):
multi_process_parallel(monkeypatch, tp_size, 1, test_target)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("pp_size", [2])
@pytest.mark.parametrize(
"test_target", [send_recv_test_worker, send_recv_tensor_dict_test_worker]
)
def test_multi_process_pipeline_parallel(
monkeypatch: pytest.MonkeyPatch,
pp_size: int,
test_target: Callable[..., Any],
):
multi_process_parallel(monkeypatch, 1, pp_size, test_target)
@multi_gpu_test(num_gpus=4)
@pytest.mark.parametrize("tp_size", [2])
@pytest.mark.parametrize("pp_size", [2])
@pytest.mark.parametrize(
"test_target",
[
send_recv_test_worker,
send_recv_tensor_dict_test_worker,
all_reduce_test_worker,
all_gather_test_worker,
broadcast_tensor_dict_test_worker,
],
)
def test_multi_process_tensor_parallel_pipeline_parallel(
tp_size: int,
pp_size: int,
test_target: Callable[..., Any],
monkeypatch: pytest.MonkeyPatch,
):
multi_process_parallel(monkeypatch, tp_size, pp_size, test_target)
-318
View File
@@ -1,318 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""WARNING: This test runs in both single-node (4 GPUs) and multi-node
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
important to set the distributed backend to "mp" to avoid Ray scheduling
all workers in a node other than the head node, which can cause the test
to fail.
"""
import json
import os
from dataclasses import dataclass
from typing import Literal, NamedTuple
import lm_eval
import pytest
import torch
from tests.utils import RemoteOpenAIServer, create_new_process_for_each_test
from vllm.config.model import RunnerOption
from vllm.logger import init_logger
from vllm.platforms import current_platform
from ..models.registry import HF_EXAMPLE_MODELS
logger = init_logger("test_context_parallel")
VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
CP_TEST_MODELS = [
# TODO support other models
# [LANGUAGE GENERATION]
"deepseek-ai/DeepSeek-V2-Lite-Chat",
"Qwen/Qwen2.5-1.5B-Instruct",
"Qwen/Qwen3.5-0.8B", # hybrid attention model
]
# GSM8K eval configuration
NUM_SHOTS = 5 # Few-shot examples
TASK = "gsm8k"
FILTER = "exact_match,strict-match"
NUM_CONCURRENT = 128
# tp accuracy with 2% buffer
MIN_ACCURACY = {
# .buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml
"deepseek-ai/DeepSeek-V2-Lite-Chat": 0.64,
# .buildkite/lm-eval-harness/configs/Qwen2.5-1.5B-Instruct.yaml
"Qwen/Qwen2.5-1.5B-Instruct": 0.52,
"Qwen/Qwen3.5-0.8B": 0.33,
}
class ParallelSetup(NamedTuple):
tp_size: int
pp_size: int
dcp_size: int
cp_kv_cache_interleave_size: int
eager_mode: bool
chunked_prefill: bool
class CPTestOptions(NamedTuple):
multi_node_only: bool
attn_backend: str | None = None
@dataclass
class CPTestSettings:
parallel_setups: list[ParallelSetup]
distributed_backends: list[str]
runner: RunnerOption
test_options: CPTestOptions
@staticmethod
def detailed(
*,
tp_base: int = 4,
pp_base: int = 1,
dcp_multipliers: list[float] | None = None,
cp_kv_cache_interleave_size: int = 1,
multi_node_only: bool = False,
runner: RunnerOption = "auto",
attn_backend: str | None = None,
):
parallel_setups = []
if dcp_multipliers is None:
dcp_multipliers = [
0.5,
]
for eager_mode_val in [False]:
for pp_multiplier in [1]:
for dcp_multiplier in dcp_multipliers:
for chunked_prefill_val in [True]:
parallel_setups.append(
ParallelSetup(
tp_size=tp_base,
pp_size=pp_multiplier * pp_base,
dcp_size=int(dcp_multiplier * tp_base),
cp_kv_cache_interleave_size=cp_kv_cache_interleave_size,
eager_mode=eager_mode_val,
chunked_prefill=chunked_prefill_val,
)
)
return CPTestSettings(
parallel_setups=parallel_setups,
distributed_backends=["mp"],
runner=runner,
test_options=CPTestOptions(
multi_node_only=multi_node_only,
attn_backend=attn_backend,
),
)
def iter_params(self, model_id: str):
opts = self.test_options
for parallel_setup in self.parallel_setups:
for backend in self.distributed_backends:
yield (
model_id,
parallel_setup,
backend,
self.runner,
opts,
)
if current_platform.is_rocm():
CP_TEXT_GENERATION_MODELS = {
"deepseek-ai/DeepSeek-V2-Lite-Chat": [
CPTestSettings.detailed(dcp_multipliers=[1]),
],
"Qwen/Qwen2.5-1.5B-Instruct": [
CPTestSettings.detailed(dcp_multipliers=[1]),
],
}
else:
CP_TEXT_GENERATION_MODELS = {
"deepseek-ai/DeepSeek-V2-Lite-Chat": [
CPTestSettings.detailed(dcp_multipliers=[1]),
CPTestSettings.detailed(
dcp_multipliers=[0.5],
cp_kv_cache_interleave_size=64,
attn_backend="FLASHMLA",
),
],
"Qwen/Qwen2.5-1.5B-Instruct": [
CPTestSettings.detailed(
cp_kv_cache_interleave_size=16, attn_backend="FLASH_ATTN"
),
CPTestSettings.detailed(
cp_kv_cache_interleave_size=16, attn_backend="FLASHINFER"
),
],
"Qwen/Qwen3.5-0.8B": [
CPTestSettings.detailed(
cp_kv_cache_interleave_size=16,
attn_backend="FLASH_ATTN",
),
],
}
def _test_cp_gsm8k(
model_id: str,
parallel_setup: ParallelSetup,
distributed_backend: str,
runner: RunnerOption,
test_options: CPTestOptions,
num_gpus_available: int,
*,
method: Literal["generate"],
is_multimodal: bool,
):
(
tp_size,
pp_size,
dcp_size,
cp_kv_cache_interleave_size,
eager_mode,
chunked_prefill,
) = parallel_setup
multi_node_only, attn_backend = test_options
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
model_info.check_transformers_version(on_fail="skip")
trust_remote_code = model_info.trust_remote_code
tokenizer_mode = model_info.tokenizer_mode
hf_overrides = model_info.hf_overrides
model_info.check_available_online(on_fail="skip")
if num_gpus_available < tp_size * pp_size:
pytest.skip(f"Need at least {tp_size} x {pp_size} GPUs")
if VLLM_MULTI_NODE and distributed_backend == "mp":
pytest.skip(
"Skipping multi-node pipeline parallel test for "
"multiprocessing distributed backend"
)
if multi_node_only and not VLLM_MULTI_NODE:
pytest.skip("Not in multi-node setting")
server_args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"bfloat16",
"--max-model-len",
"4096",
"--max-num-seqs",
"64",
]
if chunked_prefill:
server_args.append("--enable-chunked-prefill")
if eager_mode:
server_args.append("--enforce-eager")
if runner != "auto":
server_args.extend(["--runner", runner])
if trust_remote_code:
server_args.append("--trust-remote-code")
if tokenizer_mode:
server_args.extend(["--tokenizer-mode", tokenizer_mode])
if hf_overrides:
server_args.extend(["--hf-overrides", json.dumps(hf_overrides)])
server_args.extend(
[
"--tensor-parallel-size",
str(tp_size),
"--pipeline-parallel-size",
str(pp_size),
"--decode-context-parallel-size",
str(dcp_size),
"--dcp-kv-cache-interleave-size",
str(cp_kv_cache_interleave_size),
"--distributed-executor-backend",
distributed_backend,
]
)
if attn_backend:
server_args.append(f"--attention-backend={attn_backend}")
with RemoteOpenAIServer(
model_id,
server_args,
max_wait_seconds=720,
) as remote_server:
url = f"{remote_server.url_for('v1')}/completions"
model_args = (
f"model={model_id},"
f"base_url={url},"
f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False"
)
results = lm_eval.simple_evaluate(
model="local-completions",
model_args=model_args,
tasks=TASK,
num_fewshot=NUM_SHOTS,
)
# Validate accuracy is reasonable
accuracy = results["results"][TASK][FILTER]
min_accuracy = MIN_ACCURACY[model_id]
assert accuracy >= min_accuracy, (
f"TP+DCP accuracy too low: {accuracy:.3f} < {min_accuracy:.3f}"
)
@pytest.mark.parametrize(
(
"model_id",
"parallel_setup",
"distributed_backend",
"runner",
"test_options",
),
[
params
for model_id, settings in CP_TEXT_GENERATION_MODELS.items()
for setting in settings
for params in setting.iter_params(model_id)
if model_id in CP_TEST_MODELS
],
)
@create_new_process_for_each_test()
def test_cp_generation(
model_id: str,
parallel_setup: ParallelSetup,
distributed_backend: str,
runner: RunnerOption,
test_options: CPTestOptions,
num_gpus_available,
):
if (
model_id == "deepseek-ai/DeepSeek-V2-Lite-Chat"
and torch.cuda.get_device_capability() < (9, 0)
):
pytest.skip(reason="MLA+DCP requires compute capability of 9.0 or higher")
if (
model_id == "Qwen/Qwen2.5-1.5B-Instruct"
and torch.cuda.get_device_capability() != (9, 0)
):
pytest.skip(reason="GQA+DCP currently requires compute capability of 9.0")
_test_cp_gsm8k(
model_id,
parallel_setup,
distributed_backend,
runner,
test_options,
num_gpus_available,
method="generate",
is_multimodal=False,
)
@@ -1,394 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Validate production Lamport cleanup from a reachable one-epoch skew.
The fast rank publishes epoch N+1 before the slow rank runs epoch N cleanup.
Peer slots are prefilled only so each shipping kernel can finish independently;
the test then checks that slow-rank cleanup retained the real N+1 publication.
The production kernels are also seeded at the final two uint32 values to verify
that stage selection remains bounded and never reuses a stage across wrap.
Each rank's local buffer contains three all-gather stages followed by three
reduce-scatter stages::
all-gather: [stage 0][stage 1][stage 2]
reduce-scatter: [stage 0][stage 1][stage 2]
Each stage contains one payload slot per source rank.
"""
import pytest
import ray
import torch
import torch.distributed as dist
from vllm.distributed.parallel_state import get_tp_group
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter
from vllm.platforms import current_platform
from ..utils import (
ensure_model_parallel_initialized,
init_test_distributed_environment,
multi_process_parallel,
)
_SENTINEL = -2147483648
_ROWS = 32
_HIDDEN = 8
_NUM_STAGES = 3
_NUM_RANKS = 2
_FAST_RANK = 0
_SLOW_RANK = 1
_PACK_BYTES = 16
_UINT32_MODULUS = 1 << 32
_UINT32_MAX = _UINT32_MODULUS - 1
_ALL_GATHER_EPOCH_ROW = 0
_REDUCE_SCATTER_EPOCH_ROW = 1
_REDUCE_SCATTER_REGION_STAGE_OFFSET = 3
def _supports_multimem():
capability = current_platform.get_device_capability()
return (
current_platform.is_cuda() and capability is not None and capability.major >= 9
)
def _supports_sm100_or_sm103():
capability = current_platform.get_device_capability()
return current_platform.is_cuda() and capability in ((10, 0), (10, 3))
def _payload(value, device):
return torch.full((_ROWS, _HIDDEN), value, dtype=torch.bfloat16, device=device)
def _payload_bytes(value):
return value.contiguous().view(torch.uint8).flatten()
def _int32_bits(value):
"""Represent an unsigned 32-bit value in the signed epoch tensor."""
value %= _UINT32_MODULUS
return value if value < (1 << 31) else value - _UINT32_MODULUS
def _epoch_value(epoch):
return int(epoch.item()) % _UINT32_MODULUS
def _stage_slot(buffer, stage_size, region_offset, stage, src_rank, slot_size):
offset = region_offset + stage * stage_size + src_rank * slot_size
return buffer.narrow(0, offset, slot_size)
def _prepare_rank_state(
fa, epoch_row, region_offset, epoch, stage, peer_rank, peer_payload
):
"""Reset this rank's replica and seed the peer payload needed to finish."""
buffer = fa.mnnvl_buffer
all_epochs = fa.mnnvl_lamport_epochs
assert buffer is not None and all_epochs is not None
slot_size = peer_payload.nbytes
slot = _stage_slot(
buffer, fa.mnnvl_buffer_size, region_offset, stage, peer_rank, slot_size
)
buffer.view(torch.int32).fill_(_SENTINEL)
epochs = all_epochs[epoch_row]
# epochs[0] selects the stage, epochs[1] is the CTA completion counter,
# and epochs[2:5] record the valid pack count for each stage.
epochs.zero_()
epochs[0] = epoch
epochs[2:5].fill_(slot_size * _NUM_RANKS // _PACK_BYTES)
slot.copy_(_payload_bytes(peer_payload))
return buffer, slot_size
def _all_ranks_agree(checks, device):
"""Give every rank the same result so one failure cannot strand a peer."""
values = torch.tensor(checks, dtype=torch.int32, device=device)
dist.all_reduce(values, op=dist.ReduceOp.MIN)
return [bool(value) for value in values.cpu().tolist()]
def _run_all_gather_retention_case(fa, rank, local_rank, base_epoch, device):
slow_epoch = base_epoch
fast_epoch = base_epoch + 1
slow_stage = slow_epoch % _NUM_STAGES
fast_stage = fast_epoch % _NUM_STAGES
fast_local = _payload(11, device)
fast_peer = _payload(21, device)
slow_peer = _payload(31, device)
slow_local = _payload(41, device)
if rank == _FAST_RANK:
buffer, slot_size = _prepare_rank_state(
fa,
_ALL_GATHER_EPOCH_ROW,
0,
fast_epoch,
fast_stage,
_SLOW_RANK,
fast_peer,
)
else:
buffer, slot_size = _prepare_rank_state(
fa,
_ALL_GATHER_EPOCH_ROW,
0,
slow_epoch,
slow_stage,
_FAST_RANK,
slow_peer,
)
torch.accelerator.synchronize()
dist.barrier(device_ids=[local_rank])
# Fast rank publishes N+1 through the shipping all-gather kernel.
fast_output_ok = True
if rank == _FAST_RANK:
output = fa.custom_all_gather(fast_local)
assert output is not None
torch.accelerator.synchronize()
fast_output_ok = torch.equal(output, torch.cat((fast_local, fast_peer)))
dist.barrier(device_ids=[local_rank])
# Slow rank verifies that publication, then runs shipping epoch-N cleanup.
publication_ok = True
slow_output_ok = True
retained = True
if rank == _SLOW_RANK:
target = _stage_slot(
buffer,
fa.mnnvl_buffer_size,
0,
fast_stage,
_FAST_RANK,
slot_size,
)
publication_ok = torch.equal(target, _payload_bytes(fast_local))
output = fa.custom_all_gather(slow_local)
assert output is not None
torch.accelerator.synchronize()
slow_output_ok = torch.equal(output, torch.cat((slow_peer, slow_local)))
retained = torch.equal(target, _payload_bytes(fast_local))
checks = _all_ranks_agree(
[fast_output_ok, publication_ok, slow_output_ok, retained], device
)
assert all(checks[:3]), "production all-gather setup did not reach target state"
assert checks[3], "all-gather cleanup erased the next-stage publication"
def _run_reduce_scatter_retention_case(fa, rank, local_rank, base_epoch, device):
slow_epoch = base_epoch
fast_epoch = base_epoch + 1
slow_stage = slow_epoch % _NUM_STAGES
fast_stage = fast_epoch % _NUM_STAGES
region_offset = _REDUCE_SCATTER_REGION_STAGE_OFFSET * fa.mnnvl_buffer_size
fast_chunk_0 = _payload(11, device)
fast_chunk_1 = _payload(12, device)
fast_peer = _payload(21, device)
slow_chunk_0 = _payload(31, device)
slow_chunk_1 = _payload(32, device)
slow_peer = _payload(41, device)
fast_input = torch.cat((fast_chunk_0, fast_chunk_1))
slow_input = torch.cat((slow_chunk_0, slow_chunk_1))
if rank == _FAST_RANK:
buffer, slot_size = _prepare_rank_state(
fa,
_REDUCE_SCATTER_EPOCH_ROW,
region_offset,
fast_epoch,
fast_stage,
_SLOW_RANK,
fast_peer,
)
else:
buffer, slot_size = _prepare_rank_state(
fa,
_REDUCE_SCATTER_EPOCH_ROW,
region_offset,
slow_epoch,
slow_stage,
_FAST_RANK,
slow_peer,
)
torch.accelerator.synchronize()
dist.barrier(device_ids=[local_rank])
# Fast rank publishes its N+1 contribution through shipping reduce-scatter.
fast_output_ok = True
if rank == _FAST_RANK:
output = fa.custom_reduce_scatter(fast_input)
assert output is not None
torch.accelerator.synchronize()
fast_output_ok = torch.equal(output, fast_chunk_0 + fast_peer)
dist.barrier(device_ids=[local_rank])
# Slow rank verifies that contribution, then runs shipping epoch-N cleanup.
publication_ok = True
slow_output_ok = True
retained = True
if rank == _SLOW_RANK:
target = _stage_slot(
buffer,
fa.mnnvl_buffer_size,
region_offset,
fast_stage,
_FAST_RANK,
slot_size,
)
publication_ok = torch.equal(target, _payload_bytes(fast_chunk_1))
output = fa.custom_reduce_scatter(slow_input)
assert output is not None
torch.accelerator.synchronize()
slow_output_ok = torch.equal(output, slow_peer + slow_chunk_1)
retained = torch.equal(target, _payload_bytes(fast_chunk_1))
checks = _all_ranks_agree(
[fast_output_ok, publication_ok, slow_output_ok, retained], device
)
assert all(checks[:3]), "production reduce-scatter setup did not reach target state"
assert checks[3], "reduce-scatter cleanup erased the next-stage publication"
def _run_epoch_wrap_case(fa, rank, local_rank, seed_epoch, device):
"""Run both shipping collectives across the uint32 wrap boundary."""
buffer = fa.mnnvl_buffer
all_epochs = fa.mnnvl_lamport_epochs
assert buffer is not None and all_epochs is not None
buffer.view(torch.int32).fill_(_SENTINEL)
all_epochs.zero_()
all_epochs[:, 0] = _int32_bits(seed_epoch)
torch.accelerator.synchronize()
dist.barrier(device_ids=[local_rank])
ag_input = _payload(rank + 1, device)
ag_output = fa.custom_all_gather(ag_input)
assert ag_output is not None
rs_input = torch.cat(
(_payload(rank + 1, device), _payload((rank + 1) * 10, device))
)
rs_output = fa.custom_reduce_scatter(rs_input)
assert rs_output is not None
torch.accelerator.synchronize()
next_stage = (seed_epoch % _NUM_STAGES + 1) % _NUM_STAGES
checks = _all_ranks_agree(
[
torch.equal(
ag_output,
torch.cat((_payload(1, device), _payload(2, device))),
),
torch.equal(rs_output, _payload(3 if rank == 0 else 30, device)),
_epoch_value(all_epochs[_ALL_GATHER_EPOCH_ROW, 0]) == next_stage,
_epoch_value(all_epochs[_REDUCE_SCATTER_EPOCH_ROW, 0]) == next_stage,
],
device,
)
assert all(checks), "Lamport stages did not advance safely across uint32 wrap"
@ray.remote(num_gpus=1, max_calls=1)
def _run_stage_cleanup_test(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
m.setenv("VLLM_ALLREDUCE_USE_SYMM_MEM", "1")
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
fa = get_tp_group().device_communicator.ca_comm
assert fa is not None and not fa.disabled
assert fa.mnnvl_multicast_ptr
for base_epoch in range(_NUM_STAGES):
_run_all_gather_retention_case(fa, rank, rank, base_epoch, device)
_run_reduce_scatter_retention_case(fa, rank, rank, base_epoch, device)
# Old unbounded metadata stores UINT32_MAX and then repeats stage 0.
for seed_epoch in (_UINT32_MAX - 1, _UINT32_MAX):
_run_epoch_wrap_case(fa, rank, rank, seed_epoch, device)
@pytest.mark.skipif(
not _supports_multimem(),
reason="MNNVL Lamport collectives require an SM90 or newer NVIDIA GPU.",
)
def test_mnnvl_lamport_stage_cleanup(monkeypatch: pytest.MonkeyPatch):
if torch.accelerator.device_count() < 2:
pytest.skip("Need at least two GPUs to run the test.")
multi_process_parallel(monkeypatch, 2, 1, _run_stage_cleanup_test)
@ray.remote(num_gpus=1, max_calls=1)
def _run_multimem_reduce_scatter_test(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
fa = get_tp_group().device_communicator.ca_comm
assert fa is not None and not fa.disabled
assert fa.mnnvl_multimem_rs_supported
assert not fa.mnnvl_multimem_rs_initialized
assert not fa.mnnvl_multimem_rs_multicast_ptr
message_bytes = {
"mnnvl_lamport": 1 * 1024 * 1024,
"mnnvl_multimem": 20 * 1024 * 1024,
"nccl": 64 * 1024 * 1024 + 128,
}
for _ in range(2):
for backend, size_bytes in message_bytes.items():
shard_elements = size_bytes // torch.bfloat16.itemsize // tp_size
inp = torch.full(
(tp_size, shard_elements),
rank + 1,
dtype=torch.bfloat16,
device=device,
)
selected = fa._select_reduce_scatter_backend(inp)
if backend == "nccl":
assert selected is None
out = sp_reduce_scatter(inp)
else:
assert selected == backend
out = fa.custom_reduce_scatter(inp)
assert out is not None
if backend == "mnnvl_multimem":
assert fa.mnnvl_multimem_rs_initialized
assert fa.mnnvl_multimem_rs_multicast_ptr
expected = tp_size * (tp_size + 1) // 2
torch.testing.assert_close(out, torch.full_like(out, expected))
@pytest.mark.skipif(
not _supports_sm100_or_sm103(),
reason="The low-SM MNNVL reduce-scatter path requires SM100 or SM103.",
)
@pytest.mark.parametrize("tp_size", [2, 4, 8])
def test_mnnvl_multimem_reduce_scatter(monkeypatch: pytest.MonkeyPatch, tp_size: int):
if torch.accelerator.device_count() < tp_size:
pytest.skip(f"Need at least {tp_size} GPUs to run the test.")
multi_process_parallel(monkeypatch, tp_size, 1, _run_multimem_reduce_scatter_test)
-290
View File
@@ -1,290 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor
import pytest
import requests
from vllm.platforms import current_platform
from ..evals.gsm8k.gsm8k_eval import evaluate_gsm8k
from ..utils import RemoteOpenAIServer, multi_gpu_test
@pytest.fixture(autouse=True)
def cleanup_ray_between_tests():
"""Force-stop any lingering Ray processes between tests."""
subprocess.run(["ray", "stop", "--force"], timeout=30, capture_output=True)
time.sleep(5)
yield
MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
NUM_GSM8K_QUESTIONS = 256
EXPECTED_ACCURACY = 0.58
ACCURACY_TOL = 0.08
DEFAULT_ALL2ALL_BACKEND = "allgather_reducescatter"
def _send_scale_command(server: RemoteOpenAIServer, new_dp_size: int) -> bool:
url = server.url_for("scale_elastic_ep")
payload = {"new_data_parallel_size": new_dp_size}
headers = {"Content-Type": "application/json"}
try:
response = requests.post(url, json=payload, headers=headers, timeout=300)
return response.status_code == 200
except requests.exceptions.RequestException:
return False
def _traffic_loop(
server: RemoteOpenAIServer,
dp_rank: int | None,
ready: threading.Barrier,
stop: threading.Event,
finished: threading.Event,
is_probe: bool = False,
) -> list[tuple[float, float, int | None, str | None]]:
url = server.url_for("is_scaling_elastic_ep" if is_probe else "v1/completions")
payload = {"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 4}
headers = None if dp_rank is None else {"X-data-parallel-rank": str(dp_rank)}
request_payload = None if is_probe else payload
responses = []
is_ready = False
while not stop.is_set():
request_start = time.perf_counter()
phase = None
try:
response = requests.post(
url, json=request_payload, headers=headers, timeout=120
)
status_code = response.status_code
if is_probe and status_code == 200:
phase = response.json()["phase"]
except requests.exceptions.RequestException:
status_code = None
responses.append((request_start, time.perf_counter(), status_code, phase))
if status_code == 200 and (not is_probe or phase in ("idle", "completed")):
if not is_ready:
ready.wait(timeout=120)
is_ready = True
if finished.is_set():
return responses
time.sleep(0.05)
return responses
def _downtime(responses: list[tuple[float, float, int | None, str | None]]) -> float:
committing = [end for _, end, _, phase in responses if phase == "committing"]
if not committing:
return 0
recovered = next(
end
for _, end, status, phase in responses
if status == 200 and phase in ("idle", "completed") and end > committing[-1]
)
return recovered - committing[0]
def _scale_with_traffic(
server: RemoteOpenAIServer,
source_dp_size: int,
new_dp_size: int,
traffic_mode: str,
) -> None:
traffic_clients: list[int | None] = []
if traffic_mode == "light":
traffic_clients = [0]
elif traffic_mode == "heavy":
traffic_clients = [None] * source_dp_size
clients = [(None, True)] + [(rank, False) for rank in traffic_clients]
ready = threading.Barrier(len(clients) + 1)
stop = threading.Event()
finished = threading.Event()
with ThreadPoolExecutor(max_workers=len(clients)) as executor:
futures = [
executor.submit(
_traffic_loop, server, rank, ready, stop, finished, is_probe
)
for rank, is_probe in clients
]
try:
ready.wait(timeout=120)
start_time = time.perf_counter()
assert _send_scale_command(server, new_dp_size)
scale_seconds = time.perf_counter() - start_time
finished.set()
probe_result, *results = [future.result(timeout=120) for future in futures]
finally:
stop.set()
bad_statuses = {
status
for responses in [probe_result, *results]
for _, _, status, _ in responses
if status not in (200, 503)
}
assert not bad_statuses, f"traffic got unexpected statuses {bad_statuses}"
probe_committing = [
start for start, _, _, phase in probe_result if phase == "committing"
]
assert probe_committing, "Scaling probe did not observe commit"
assert not results or any(
status == 200
and start_time <= request_start
and request_end < probe_committing[0]
for responses in results
for request_start, request_end, status, _ in responses
), "No request completed successfully during preparation"
print(
f"[Elastic EP timing][{source_dp_size}->{new_dp_size}]"
f"[traffic={traffic_mode}] "
f"scale_seconds={scale_seconds:.3f} "
f"downtime_seconds={_downtime(probe_result):.3f}"
)
def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
assert server.port is not None
result = evaluate_gsm8k(
num_questions=NUM_GSM8K_QUESTIONS,
host=f"http://{server.host}",
port=server.port,
)
accuracy = result["accuracy"]
print(
f"[{stage}] GSM8K accuracy: {accuracy:.3f} "
f"({result['num_questions']} questions)"
)
assert accuracy >= EXPECTED_ACCURACY, (
f"[{stage}] GSM8K accuracy {accuracy:.3f} is below "
f"expected threshold {EXPECTED_ACCURACY}"
)
return accuracy
def _base_serve_args(
dp_size: int = 2,
max_dp_size: int = 4,
enforce_eager: bool = False,
all2all_backend: str = DEFAULT_ALL2ALL_BACKEND,
) -> list[str]:
args = [
"--trust-remote-code",
"--tensor-parallel-size",
"1",
"--gpu-memory-utilization",
"0.8",
"--max-model-len",
"4096",
"--enable-expert-parallel",
"--all2all-backend",
all2all_backend,
"--enable-elastic-ep",
"--elastic-ep-max-dp-size",
str(max_dp_size),
"--enable-eplb",
"--eplb-config.num_redundant_experts",
"0",
"--eplb-config.use_async",
"true",
"--eplb-config.step_interval",
"300",
"--eplb-config.window_size",
"5",
"--data-parallel-backend",
"ray",
"--data-parallel-size",
str(dp_size),
"--api-server-count",
"1",
"--disable-access-log-for-endpoints",
"/is_scaling_elastic_ep",
]
leader_address = os.environ.get("LEADER_ADDRESS")
if leader_address:
args.extend(["--data-parallel-address", leader_address])
if enforce_eager:
args.append("--enforce-eager")
return args
@pytest.mark.parametrize(
("enforce_eager", "traffic_mode", "all2all_backend"),
[
pytest.param(True, "none", DEFAULT_ALL2ALL_BACKEND, id="enforce_eager_none"),
pytest.param(True, "light", DEFAULT_ALL2ALL_BACKEND, id="enforce_eager_light"),
pytest.param(True, "heavy", DEFAULT_ALL2ALL_BACKEND, id="enforce_eager_heavy"),
pytest.param(
False,
"heavy",
DEFAULT_ALL2ALL_BACKEND,
id="cuda_graphs_heavy",
),
pytest.param(False, "heavy", "nixl_ep", id="cuda_graphs_heavy_nixl_ep"),
],
)
@multi_gpu_test(num_gpus=4)
def test_elastic_ep_scaling(
enforce_eager: bool, traffic_mode: str, all2all_backend: str
):
from vllm.distributed.eplb.eplb_communicator import has_nixl
from vllm.utils.import_utils import has_nixl_ep
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
if current_platform.is_xpu():
pytest.skip("Elastic EP with NIXL is not supported on XPU")
if all2all_backend == "nixl_ep" and not has_nixl_ep():
pytest.skip("NIXL EP is not installed")
initial_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_INITIAL_DP", "2"))
target_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_TARGET_DP", "4"))
assert target_dp_size > initial_dp_size
vllm_serve_args = _base_serve_args(
initial_dp_size, target_dp_size, enforce_eager, all2all_backend
)
with RemoteOpenAIServer(
MODEL_NAME,
vllm_serve_args,
max_wait_seconds=1200,
) as server:
initial_accuracy = _run_gsm8k_eval(server, "Initial")
_scale_with_traffic(server, initial_dp_size, target_dp_size, traffic_mode)
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up")
assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
f"Scale up accuracy {scale_up_accuracy:.3f} dropped more than "
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
)
_scale_with_traffic(server, target_dp_size, initial_dp_size, traffic_mode)
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down")
assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (
f"Scale down accuracy {scale_down_accuracy:.3f} dropped more than "
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
)
print("\nAccuracy Summary:")
print(f" Initial: {initial_accuracy:.3f}")
print(
f" Scale up: {scale_up_accuracy:.3f} "
f"(diff: {scale_up_accuracy - initial_accuracy:+.3f})"
)
print(
f" Scale down: {scale_down_accuracy:.3f} "
f"(diff: {scale_down_accuracy - initial_accuracy:+.3f})"
)
print(f" Tolerance: {ACCURACY_TOL:.3f}")
-518
View File
@@ -1,518 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
import torch
from vllm.distributed.eplb.eplb_state import (
_compute_eplb_load_stats, # pyright: ignore[reportPrivateUsage]
compute_logical_maps,
)
from vllm.distributed.eplb.policy.default import DefaultEplbPolicy
from vllm.platforms import current_platform
def test_eplb_load_stats_reduce_across_ranks():
num_tokens_per_rank = torch.tensor(
[
[100, 0, 0, 0],
[100, 0, 0, 0],
],
dtype=torch.float32,
)
avg_tokens, max_tokens = _compute_eplb_load_stats(num_tokens_per_rank)
assert avg_tokens.item() == 50
assert max_tokens.item() == 200
assert (avg_tokens / max_tokens).item() == 0.25
def test_basic_rebalance():
"""Test basic rebalancing functionality."""
# Example from https://github.com/deepseek-ai/eplb
weight = torch.tensor(
[
[90, 132, 40, 61, 104, 165, 39, 4, 73, 56, 183, 86],
[20, 107, 104, 64, 19, 197, 187, 157, 172, 86, 16, 27],
]
)
num_layers = weight.shape[0]
num_replicas = 16
num_groups = 4
num_nodes = 2
num_gpus = 8
phy2log = DefaultEplbPolicy.rebalance_experts(
weight, num_replicas, num_groups, num_nodes, num_gpus
)
log2phy, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
# Verify output shapes
assert phy2log.shape == (
2,
16,
), f"Expected `phy2log` shape (2, 16), got {phy2log.shape}"
assert log2phy.shape[0] == 2, (
f"Expected `log2phy` first dimension 2, got {log2phy.shape[0]}"
)
assert log2phy.shape[1] == 12, (
f"Expected `log2phy` second dimension 12, got {log2phy.shape[1]}"
)
assert logcnt.shape == (
2,
12,
), f"Expected `logcnt` shape (2, 12), got {logcnt.shape}"
# Verify physical to logical expert mapping range is correct
assert torch.all(phy2log >= 0) and torch.all(phy2log < 12), (
"Physical to logical mapping should be in range [0, 12)"
)
# Verify expert count reasonableness
assert torch.all(logcnt >= 1), "Each logical expert should have at least 1 replica"
assert torch.sum(logcnt, dim=1).sum() == num_replicas * num_layers, (
f"Total replicas should be {num_replicas * num_layers}"
)
# Verify expected output
expected_phy2log = torch.tensor(
[
[5, 6, 5, 7, 8, 4, 3, 4, 10, 9, 10, 2, 0, 1, 11, 1],
[7, 10, 6, 8, 6, 11, 8, 9, 2, 4, 5, 1, 5, 0, 3, 1],
]
)
assert torch.all(phy2log == expected_phy2log)
expected_logcnt = torch.tensor(
[[1, 2, 1, 1, 2, 2, 1, 1, 1, 1, 2, 1], [1, 2, 1, 1, 1, 2, 2, 1, 2, 1, 1, 1]]
)
assert torch.all(logcnt == expected_logcnt)
def test_single_gpu_case():
"""Test single GPU case."""
weight = torch.tensor([[10, 20, 30, 40]])
num_replicas = 4
num_groups = 1
num_nodes = 1
num_gpus = 1
phy2log = DefaultEplbPolicy.rebalance_experts(
weight, num_replicas, num_groups, num_nodes, num_gpus
)
log2phy, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
# Verify shapes
assert phy2log.shape == (1, 4)
assert log2phy.shape[0] == 1
assert log2phy.shape[1] == 4
assert logcnt.shape == (1, 4)
# Verify all logical experts are mapped
assert set(phy2log[0].tolist()) == {0, 1, 2, 3}
def test_equal_weights():
"""Test case with equal weights."""
weight = torch.tensor([[50, 50, 50, 50, 50, 50, 50, 50]])
num_replicas = 8
num_groups = 2
num_nodes = 2
num_gpus = 4
phy2log = DefaultEplbPolicy.rebalance_experts(
weight, num_replicas, num_groups, num_nodes, num_gpus
)
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
# Verify shapes
assert phy2log.shape == (1, 8)
assert logcnt.shape == (1, 8)
# With equal weights, each expert should have exactly one replica
assert torch.all(logcnt == 1), (
"With equal weights and no replication, "
"each expert should have exactly 1 replica"
)
def test_extreme_weight_imbalance():
"""Test extreme weight imbalance case."""
weight = torch.tensor([[1000, 1, 1, 1, 1, 1, 1, 1]])
num_replicas = 12
num_groups = 2
num_nodes = 2
num_gpus = 4
phy2log = DefaultEplbPolicy.rebalance_experts(
weight, num_replicas, num_groups, num_nodes, num_gpus
)
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
# Verify shapes
assert phy2log.shape == (1, 12)
assert logcnt.shape == (1, 8)
# Expert with highest weight (index 0) should have more replicas
assert logcnt[0, 0] > logcnt[0, 1], (
"Expert with highest weight should have more replicas"
)
def test_multiple_layers():
"""Test multiple layers case."""
weight = torch.tensor(
[
[10, 20, 30, 40, 50, 60], # First layer
[60, 50, 40, 30, 20, 10], # Second layer (opposite weight pattern)
[25, 25, 25, 25, 25, 25], # Third layer (equal weights)
]
)
num_replicas = 8
num_groups = 2
num_nodes = 2
num_gpus = 4
phy2log = DefaultEplbPolicy.rebalance_experts(
weight, num_replicas, num_groups, num_nodes, num_gpus
)
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
# Verify shapes
assert phy2log.shape == (3, 8)
assert logcnt.shape == (3, 6)
# Verify expert allocation is reasonable for each layer
for layer in range(3):
assert torch.all(phy2log[layer] >= 0) and torch.all(phy2log[layer] < 6), (
f"Layer {layer} physical to logical mappingshould be in range [0, 6)"
)
assert torch.sum(logcnt[layer]) == num_replicas, (
f"Layer {layer} total replicas should be {num_replicas}"
)
def test_parameter_validation():
"""Test parameter validation."""
weight = torch.tensor([[10, 20, 30, 40]])
# Test non-divisible case - this should handle normally without throwing
# errors because the function will fall back to global load balancing
# strategy
phy2log = DefaultEplbPolicy.rebalance_experts(weight, 8, 3, 2, 4)
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
assert phy2log.shape == (1, 8)
assert logcnt.shape == (1, 4)
# Test cases that will actually cause errors:
# num_physical_experts not divisible by num_gpus
with pytest.raises(AssertionError):
DefaultEplbPolicy.rebalance_experts(weight, 7, 2, 2, 4) # 7 not divisible by 4
def test_small_scale_hierarchical():
"""Test small-scale hierarchical load balancing."""
weight = torch.tensor(
[
[100, 50, 200, 75, 150, 25, 300, 80], # 8 experts
]
)
num_replicas = 12
num_groups = 4 # 4 groups, 2 experts each
num_nodes = 2 # 2 nodes
num_gpus = 4 # 4 GPUs
phy2log = DefaultEplbPolicy.rebalance_experts(
weight, num_replicas, num_groups, num_nodes, num_gpus
)
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
# Verify basic constraints
assert phy2log.shape == (1, 12)
assert logcnt.shape == (1, 8)
assert torch.sum(logcnt) == num_replicas
assert torch.all(logcnt >= 1)
# Expert with highest weight should have more replicas
max_weight_expert = torch.argmax(weight[0])
assert logcnt[0, max_weight_expert] >= 2, (
"Highest weight expert should have multiple replicas"
)
def test_global_load_balance_fallback():
"""Test global load balancing fallback case."""
# When num_groups % num_nodes != 0, should fall back to global load
# balancing
weight = torch.tensor([[10, 20, 30, 40, 50, 60]])
num_replicas = 8
num_groups = 3 # Cannot be divided evenly by num_nodes=2
num_nodes = 2
num_gpus = 4
phy2log = DefaultEplbPolicy.rebalance_experts(
weight, num_replicas, num_groups, num_nodes, num_gpus
)
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
# Should work normally, just using global load balancing strategy
assert phy2log.shape == (1, 8)
assert logcnt.shape == (1, 6)
assert torch.sum(logcnt) == num_replicas
@pytest.mark.parametrize("device", ["cpu", "cuda", "xpu"])
def test_device_compatibility(device):
"""Test device compatibility."""
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA not available")
if device == "xpu" and not current_platform.is_xpu():
pytest.skip("XPU not available")
weight = torch.tensor([[10, 20, 30, 40]], device=device)
num_replicas = 6
num_groups = 2
num_nodes = 1
num_gpus = 2
phy2log = DefaultEplbPolicy.rebalance_experts(
weight, num_replicas, num_groups, num_nodes, num_gpus
)
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
# Function will convert to CPU internally, but should handle different
# device inputs normally
assert phy2log.shape == (1, 6)
assert logcnt.shape == (1, 4)
def test_additional_cases():
"""Test more edge cases and different parameter combinations."""
# Test case 1: Large-scale distributed setup
weight1 = torch.tensor(
[[50, 100, 75, 120, 90, 60, 80, 110, 40, 70, 95, 85, 65, 55, 45, 35]]
)
phy2log1 = DefaultEplbPolicy.rebalance_experts(weight1, 24, 8, 4, 8)
_, logcnt1 = compute_logical_maps(phy2log1, weight1.shape[-1])
assert phy2log1.shape == (1, 24)
assert logcnt1.shape == (1, 16)
assert torch.sum(logcnt1) == 24
# Test case 2: Different weight distributions
weight2 = torch.tensor(
[
[200, 150, 100, 50, 25, 12], # Decreasing weights
[12, 25, 50, 100, 150, 200], # Increasing weights
]
)
phy2log2 = DefaultEplbPolicy.rebalance_experts(weight2, 10, 3, 1, 2)
_, logcnt2 = compute_logical_maps(phy2log2, weight2.shape[-1])
assert phy2log2.shape == (2, 10)
assert logcnt2.shape == (2, 6)
# Verify high-weight experts have more replicas
for layer in range(2):
max_weight_idx = torch.argmax(weight2[layer])
assert logcnt2[layer, max_weight_idx] >= 2
def test_compute_logical_maps_with_negative_indices():
"""Test that compute_logical_maps correctly handles physical slots containing
-1 (unused slots).
"""
# 2 layers, 6 physical slots, 4 logical experts.
# Slots 2 and 5 are unused (-1).
phy2log = torch.tensor(
[
[0, 1, -1, 2, 3, -1],
[3, -1, 2, 1, 0, -1],
]
)
num_layers = 2
num_logical_experts = 4
log2phy, logcnt = compute_logical_maps(phy2log, num_logical_experts)
assert logcnt.shape == (num_layers, num_logical_experts)
assert log2phy.shape == (num_layers, num_logical_experts, 1)
expected_logcnt = torch.ones(num_layers, num_logical_experts, dtype=phy2log.dtype)
assert torch.all(logcnt == expected_logcnt), (
f"Expected that all replica counts == 1, got {logcnt}"
)
assert torch.all(log2phy >= 0), (
"log2phy should only contain valid physical indices, not -1"
)
assert log2phy[0, 0, 0] == 0
assert log2phy[0, 1, 0] == 1
assert log2phy[0, 2, 0] == 3
assert log2phy[0, 3, 0] == 4
if __name__ == "__main__":
weight = torch.tensor(
[
[90, 132, 40, 61, 104, 165, 39, 4, 73, 56, 183, 86],
[20, 107, 104, 64, 19, 197, 187, 157, 172, 86, 16, 27],
]
)
num_replicas = 16
num_groups = 4
num_nodes = 2
num_gpus = 8
phy2log = DefaultEplbPolicy.rebalance_experts(
weight, num_replicas, num_groups, num_nodes, num_gpus
)
print(phy2log)
test_basic_rebalance()
def _make_phy_replicas_idx_from_phy2log(phy2log: np.ndarray) -> np.ndarray:
"""Create replicas indices mapping from phy2log."""
pr = np.zeros_like(phy2log, dtype=np.int64)
for layer in range(phy2log.shape[0]):
seen: dict[int, int] = {}
row = phy2log[layer].tolist()
for i, expert in enumerate(row):
r = seen.get(expert, 0)
pr[layer, i] = r
seen[expert] = r + 1
return pr
def _validate_intragpu_rearrangement(
old_global_expert_indices: np.ndarray,
new_phy2log: np.ndarray,
new_phy_replicas_idx: np.ndarray,
post_phy2log: np.ndarray,
post_phy_replicas_idx: np.ndarray,
num_ranks: int,
slots_per_gpu: int,
):
# Per-GPU checks
for gpu_idx in range(num_ranks):
start = gpu_idx * slots_per_gpu
end = start + slots_per_gpu
old_seg = old_global_expert_indices[0, start:end]
new_seg = new_phy2log[0, start:end]
new_rnk = new_phy_replicas_idx[0, start:end]
post_seg = post_phy2log[0, start:end]
post_rnk = post_phy_replicas_idx[0, start:end]
# Pairwise equality for (expert, rank) pairs to ensure nothing is lost
def sorted_pairs(seg, rnk):
pairs = list(zip(seg.tolist(), rnk.tolist()))
pairs.sort()
return pairs
assert sorted_pairs(post_seg, post_rnk) == sorted_pairs(new_seg, new_rnk), (
f"Per-GPU pairs of (expert,rank) must match new mapping for GPU {gpu_idx}"
)
# For experts that remain on the same GPU, the old slot is preserved
# for at least one occurrence; rank at that slot must be valid for that expert
old_list = old_seg.tolist()
new_list = new_seg.tolist()
post_list = post_seg.tolist()
remained = set(old_list) & set(new_list)
new_ranks_for_expert: dict[int, list[int]] = {}
for v, r in zip(new_list, new_rnk.tolist()):
new_ranks_for_expert.setdefault(v, []).append(r)
for expert in remained:
old_pos = old_list.index(expert)
assert post_list[old_pos] == expert, (
f"Expert {expert} on GPU {gpu_idx} should stay at old slot {old_pos}"
)
# Rank at preserved slot must be one of the ranks
# the expert has in new mapping
assert post_rnk.tolist()[old_pos] in new_ranks_for_expert[expert], (
f"Rank for expert {expert} at preserved slot on GPU {gpu_idx} "
"must come from new mapping"
)
@pytest.mark.parametrize(
"num_ranks, slots_per_gpu, old_phy2log, new_phy2log",
[
pytest.param(
# Setup: 2 GPUs, 4 slots each, 1 layer
# Old mapping: GPU0 -> [0,1,2,3], GPU1 -> [4,5,6,7]
# New mapping shuffles within GPU0 and brings 4,5 into GPU0.
# GPU0 new -> [1,5,0,4]; GPU1 new -> [6,2,7,3]
2,
4,
np.array([[0, 1, 2, 3, 4, 5, 6, 7]]),
np.array([[1, 5, 0, 4, 6, 2, 7, 3]]),
id="simple",
),
pytest.param(
# Setup: 2 GPUs, 5 slots each (total 10 physical experts), 1 layer
# Old mapping:
# GPU0 -> [0, 1, 0, 2, 3] (expert 0 duplicated)
# GPU1 -> [4, 5, 6, 1, 2]
# New mapping reorders within GPUs and moves some experts across GPUs,
# while still including duplicates:
# GPU0 new -> [0, 5, 4, 0, 1] (expert 0 duplicated, 4/5 incoming)
# GPU1 new -> [6, 2, 3, 2, 1] (expert 2 duplicated)
2,
5,
np.array([[0, 1, 0, 2, 3, 4, 5, 6, 1, 2]]),
np.array([[0, 5, 4, 0, 1, 6, 2, 3, 2, 1]]),
id="duplicates",
),
pytest.param(
# Setup: 3 GPUs, 4 slots each (total 12 physical experts), 1 layer
# Old mapping:
# GPU0 -> [0, 1, 2, 3]
# GPU1 -> [0, 1, 2, 3]
# GPU2 -> [0, 1, 2, 3]
# New mapping decides to use one expert on 2 GPUs and shuffles
# experts on the third GPU,
# GPU0 new -> [0, 0, 0, 0]
# GPU1 new -> [0, 0, 0, 0]
# GPU2 new -> [1, 2, 3, 0]
3,
4,
np.array([[0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3]]),
np.array([[0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 0]]),
id="skewed_expert",
),
],
)
def test_preserve_intragpu_slots(
num_ranks: int,
slots_per_gpu: int,
old_phy2log: torch.Tensor,
new_phy2log: torch.Tensor,
):
"""Experts that stay on a GPU keep their old slots; incoming not lost."""
phy_replicas_idx = _make_phy_replicas_idx_from_phy2log(new_phy2log)
post_phy2log = DefaultEplbPolicy.preserve_intragpu_slots(
new_phy2log, num_ranks, old_phy2log
)
post_phy_replicas_idx = _make_phy_replicas_idx_from_phy2log(post_phy2log)
# Shapes preserved
assert post_phy2log.shape == new_phy2log.shape
assert post_phy_replicas_idx.shape == phy_replicas_idx.shape
_validate_intragpu_rearrangement(
old_phy2log,
new_phy2log,
phy_replicas_idx,
post_phy2log,
post_phy_replicas_idx,
num_ranks,
slots_per_gpu,
)
-104
View File
@@ -1,104 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
import time
import pytest
import torch
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream
from vllm.platforms import current_platform
def test_wait_blocks_until_record():
event = CpuGpuEvent()
record_stream = torch.Stream()
wait_stream = torch.Stream()
wait_returned = threading.Event()
def waiter():
event.wait(stream=wait_stream)
wait_returned.set()
t = threading.Thread(target=waiter)
t.start()
time.sleep(0.05)
assert not wait_returned.is_set(), "wait() returned before record() was called"
event.record(stream=record_stream)
t.join(timeout=5.0)
assert not event._recorded.is_set()
def test_reuse_across_multiple_cycles():
wrapper = CpuGpuEvent()
record_stream = torch.Stream()
wait_stream = torch.Stream()
NUM_CYCLES = 8
completed_cycles = []
barriers = [threading.Barrier(2) for _ in range(NUM_CYCLES)]
def waiter():
for i in range(NUM_CYCLES):
wrapper.wait(stream=wait_stream)
completed_cycles.append(True)
barriers[i].wait()
t = threading.Thread(target=waiter)
t.start()
for i in range(NUM_CYCLES):
wrapper.record(stream=record_stream)
barriers[i].wait()
t.join(timeout=10.0)
assert len(completed_cycles) == NUM_CYCLES
@pytest.mark.parametrize("device", ["cuda", "xpu"])
def test_producer_consumer(device):
"""This test uses the CpuGpuEvent to synchronize reads and writes to/from
a shared GPU tensor on multiple CPU threads.
"""
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA not available")
if device == "xpu" and not current_platform.is_xpu():
pytest.skip("XPU not available")
worker_stream = torch.Stream()
# Create a single element counter that will be shared between two threads
buf = torch.zeros(1, device=device)
NUM_ROUNDS = 5
ready_cpu = [threading.Event() for _ in range(NUM_ROUNDS)]
events = [CpuGpuEvent() for _ in range(NUM_ROUNDS)]
errors: list[str] = []
# For each round, the worker thread (writer) sets the counter in buf and waits for
# the main thread to read it.
def worker():
for i in range(NUM_ROUNDS):
if i > 0:
events[i - 1].wait(stream=worker_stream)
with device_stream(worker_stream):
buf.fill_(float(i + 1))
worker_stream.synchronize()
ready_cpu[i].set()
t = threading.Thread(target=worker)
t.start()
for i in range(NUM_ROUNDS):
ready_cpu[i].wait()
snapshot = buf.clone()
events[i].record()
val = snapshot.item()
if val != float(i + 1):
errors.append(f"round {i}: expected {i + 1:.1f}, got {val:.1f}")
t.join(timeout=10.0)
assert not errors, f"Buffer ordering errors: {errors}"
-894
View File
@@ -1,894 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
import pytest
import torch
import torch.distributed
import vllm.distributed.eplb.eplb_communicator as eplb_comm
import vllm.utils.gpu_sync_debug as gsd
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed.eplb.eplb_communicator import (
create_eplb_communicator,
has_nixl,
)
from vllm.distributed.eplb.rebalance_execute import (
move_from_buffer,
rearrange_expert_weights_inplace,
transfer_layer,
)
from vllm.distributed.parallel_state import (
ensure_model_parallel_initialized,
get_tp_group,
)
from vllm.platforms import current_platform
from .eplb_utils import distributed_run, set_env_vars_and_device
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_gloo_receive_staging_does_not_force_gpu_sync(monkeypatch):
"""Gloo receives must reach the GPU without an implicit pageable copy."""
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error")
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
gsd._install_copy_checkers()
monkeypatch.setattr(eplb_comm, "is_local_first_rank", lambda: False)
monkeypatch.setattr(
eplb_comm,
"P2POp",
lambda op, tensor, peer=None, group=None, tag=0, group_peer=None: tensor,
)
def receive(tensors):
for tensor in tensors:
tensor.fill_(7)
return []
monkeypatch.setattr(eplb_comm, "batch_isend_irecv", receive)
communicator = eplb_comm.TorchDistGlooStagedEplbCommunicator(cpu_group=None)
dst = torch.empty(32, device="cuda")
communicator.add_recv([dst], src_rank=1, expert_id=0)
gsd.with_gpu_sync_check(communicator.execute)()
torch.testing.assert_close(dst.cpu(), torch.full((32,), 7.0))
def create_expert_indices_with_redundancy(
num_layers: int,
num_logical_experts: int,
total_physical_experts: int,
redundancy_config: list[int], # redundancy for each logical expert
) -> torch.Tensor:
"""Create expert indices with redundancy.
Args:
num_layers: number of layers
num_logical_experts: number of logical experts
total_physical_experts: total number of physical experts
redundancy_config: redundancy for each logical expert
Returns:
indices: Shape (num_layers, total_physical_experts)
"""
assert sum(redundancy_config) == total_physical_experts
assert len(redundancy_config) == num_logical_experts
indices = torch.zeros(num_layers, total_physical_experts, dtype=torch.long)
for layer in range(num_layers):
physical_pos = 0
for logical_expert_id, redundancy in enumerate(redundancy_config):
for _ in range(redundancy):
indices[layer, physical_pos] = logical_expert_id
physical_pos += 1
# Shuffle the indices at dim 1
for layer in range(num_layers):
indices[layer] = indices[layer][torch.randperm(indices.shape[1])]
return indices
def create_expert_weights(
num_layers: int,
num_local_experts: int,
hidden_sizes: list[int],
rank: int,
device: torch.device,
physical_to_logical_mapping: torch.Tensor,
) -> list[list[torch.Tensor]]:
"""Create fake expert weights tensor for testing.
Use `arange` to generate predictable weights values, based on logical
expert ID.
All replicas of the same logical expert should have the same weights.
Args:
physical_to_logical_mapping: Shape (num_layers, num_local_experts)
mapping[layer, physical_pos] = logical_expert_id
"""
expert_weights = []
for layer in range(num_layers):
layer_weights = []
for weight_idx, hidden_size in enumerate(hidden_sizes):
weight_tensor = torch.zeros(
num_local_experts, hidden_size, device=device, dtype=torch.float32
)
for local_expert in range(num_local_experts):
# Get the logical expert ID for this physical expert
global_pos = rank * num_local_experts + local_expert
logical_expert_id = physical_to_logical_mapping[
layer, global_pos
].item()
# Generate weights based on logical expert ID
# (so that all replicas of the same logical expert have the
# same weights)
base_value = logical_expert_id * 1000 + layer * 100 + weight_idx * 10
weight_tensor[local_expert] = torch.arange(
base_value,
base_value + hidden_size,
device=device,
dtype=torch.float32,
)
layer_weights.append(weight_tensor)
expert_weights.append(layer_weights)
return expert_weights
def create_redundancy_config(
num_logical_experts: int,
num_physical_experts: int,
) -> list[int]:
"""Create a redundancy configuration."""
redundancy_config = [1] * num_logical_experts
remaining = num_physical_experts - num_logical_experts
# Randomly assign the remaining physical experts to the logical experts
for _ in range(remaining):
redundancy_config[random.choice(range(num_logical_experts))] += 1
return redundancy_config
def verify_expert_weights_after_shuffle(
expert_weights: list[list[torch.Tensor]],
new_indices: torch.Tensor,
hidden_sizes: list[int],
ep_rank: int,
num_local_experts: int,
) -> bool:
"""Verify the weights after shuffling are correct."""
num_layers = len(expert_weights)
ok = True
for layer in range(num_layers):
for weight_idx, hidden_size in enumerate(hidden_sizes):
weight_tensor = expert_weights[layer][weight_idx]
for local_expert in range(num_local_experts):
# Calculate the global expert ID for this local expert
global_pos = ep_rank * num_local_experts + local_expert
expected_logical_expert = new_indices[layer, global_pos].item()
# Check if the weights are correct
actual_weights = weight_tensor[local_expert]
expected_base = (
expected_logical_expert * 1000 + layer * 100 + weight_idx * 10
)
expected_weights = torch.arange(
expected_base,
expected_base + hidden_size,
device=actual_weights.device,
dtype=actual_weights.dtype,
)
if not torch.equal(actual_weights, expected_weights):
ok = False
actual_head = actual_weights[:8].detach().cpu().tolist()
expected_head = expected_weights[:8].detach().cpu().tolist()
print(
"verify_expert_weights_after_shuffle failed: "
f"rank={ep_rank}, "
f"layer={layer}, weight_idx={weight_idx}, "
f"local_expert={local_expert}, "
f"expected_logical_expert={expected_logical_expert}, "
f"actual_head={actual_head}, expected_head={expected_head}",
flush=True,
)
return ok
def verify_redundant_experts_have_same_weights(
expert_weights: list[list[torch.Tensor]],
indices: torch.Tensor,
hidden_sizes: list[int],
ep_rank: int,
world_size: int,
num_local_experts: int,
cpu_group,
) -> bool:
"""Verify that all replicas of the same logical expert have the same weights."""
num_layers = len(expert_weights)
total_physical_experts = world_size * num_local_experts
# XPU: gloo does not support XPU tensors, so gather on CPU then move back.
# CUDA: gloo supports CUDA tensors directly, keep original device.
use_cpu_gather = current_platform.is_xpu()
ok = True
for layer in range(num_layers):
# Collect weights for all physical experts for each weight matrix
all_weights: list[torch.Tensor] = []
for weight_idx, hidden_size in enumerate(hidden_sizes):
orig_device = expert_weights[layer][weight_idx].device
if use_cpu_gather:
local_weights = expert_weights[layer][weight_idx].cpu()
gather_device = "cpu"
else:
local_weights = expert_weights[layer][weight_idx]
gather_device = orig_device
# Create tensor to store all expert weights
# Shape: [total_physical_experts, hidden_size]
gathered_weights = torch.zeros(
total_physical_experts,
hidden_size,
device=gather_device,
dtype=local_weights.dtype,
)
# Split tensor along dim 0 into a list for all_gather
gathered_weights_list = torch.chunk(gathered_weights, world_size, dim=0)
torch.distributed.all_gather(
list(gathered_weights_list),
local_weights,
group=cpu_group,
)
if use_cpu_gather:
gathered_weights = gathered_weights.to(orig_device)
all_weights.append(gathered_weights)
# Verify that all replicas of the same logical expert have the same
# weights
logical_expert_weights: dict[int, dict[int, torch.Tensor]] = {}
for physical_pos in range(total_physical_experts):
logical_expert_id = int(indices[layer, physical_pos].item())
if logical_expert_id not in logical_expert_weights:
# First time encountering this logical expert, save its weights
logical_expert_weights[logical_expert_id] = {
weight_idx: all_weights[weight_idx][physical_pos]
for weight_idx in range(len(hidden_sizes))
}
else:
# Verify that current physical expert's weights match the
# previously saved logical expert weights
for weight_idx in range(len(hidden_sizes)):
if not torch.equal(
all_weights[weight_idx][physical_pos],
logical_expert_weights[logical_expert_id][weight_idx],
):
ok = False
actual_head = (
all_weights[weight_idx][physical_pos][:8]
.detach()
.cpu()
.tolist()
)
reference_head = (
logical_expert_weights[logical_expert_id][weight_idx][:8]
.detach()
.cpu()
.tolist()
)
print(
"verify_redundant_experts_have_same_weights failed: "
f"rank={ep_rank}, "
f"layer={layer}, weight_idx={weight_idx}, "
f"logical_expert={logical_expert_id}, "
f"physical_pos={physical_pos}, "
f"actual_head={actual_head}, "
f"reference_head={reference_head}",
flush=True,
)
return ok
def assert_verification_synced(local_ok: bool, msg: str, cpu_group) -> None:
if current_platform.is_xpu():
# XPU: default group backend (gloo) does not support XPU tensors;
# use CPU tensor with cpu_group instead.
ok_tensor = torch.tensor(
[1 if local_ok else 0], device="cpu", dtype=torch.int32
)
torch.distributed.all_reduce(
ok_tensor, op=torch.distributed.ReduceOp.MIN, group=cpu_group
)
else:
# CUDA: nccl (default group) supports device tensors directly.
device = torch.accelerator.current_accelerator()
ok_tensor = torch.tensor(
[1 if local_ok else 0], device=device, dtype=torch.int32
)
torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN)
assert bool(ok_tensor.item()), msg
def create_eplb_communicator_or_raise(
*, group_coordinator, backend, expert_weights, expert_buffer
):
try:
return create_eplb_communicator(
group_coordinator=group_coordinator,
backend=backend,
expert_weights=expert_weights,
expert_buffer=expert_buffer,
)
except Exception as exc:
raise RuntimeError(
f"Failed to create EPLB communicator for backend={backend}: {exc}"
) from exc
def _test_async_transfer_layer_without_mtp_worker(
env,
world_size: int,
num_layers: int,
num_local_experts: int,
num_logical_experts: int,
eplb_communicator: str,
) -> None:
set_env_vars_and_device(env)
vllm_config = VllmConfig()
vllm_config.parallel_config.tensor_parallel_size = world_size
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
)
ep_group_coordinator = get_tp_group()
ep_group = ep_group_coordinator.device_group
cpu_group = ep_group_coordinator.cpu_group
ep_rank = torch.distributed.get_rank()
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{ep_rank}")
total_physical_experts = world_size * num_local_experts
hidden_sizes = [16, 32]
redundancy_config = create_redundancy_config(
num_logical_experts,
total_physical_experts,
)
old_indices = create_expert_indices_with_redundancy(
num_layers,
num_logical_experts,
total_physical_experts,
redundancy_config,
)
new_redundancy_config = create_redundancy_config(
num_logical_experts,
total_physical_experts,
)
new_indices = create_expert_indices_with_redundancy(
num_layers,
num_logical_experts,
total_physical_experts,
new_redundancy_config,
)
expert_weights = create_expert_weights(
num_layers,
num_local_experts,
hidden_sizes,
ep_rank,
device,
old_indices,
)
old_indices_cpu = old_indices.cpu()
new_indices_cpu = new_indices.cpu()
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
stream = torch.Stream(device=device)
communicator = create_eplb_communicator_or_raise(
group_coordinator=ep_group_coordinator,
backend=eplb_communicator,
expert_weights=expert_weights,
expert_buffer=expert_buffer,
)
communicator.set_stream(stream)
for layer_idx in range(num_layers):
transfer_metadata = transfer_layer(
old_layer_indices=old_indices_cpu[layer_idx],
new_layer_indices=new_indices_cpu[layer_idx],
expert_weights=expert_weights[layer_idx],
expert_weights_buffer=expert_buffer,
ep_group=ep_group,
communicator=communicator,
stream=stream,
layer_idx=layer_idx,
)
stream.synchronize()
move_from_buffer(
expert_weights=expert_weights[layer_idx],
expert_weights_buffers=expert_buffer,
transfer_metadata=transfer_metadata,
new_indices=new_indices_cpu[layer_idx].numpy(),
ep_rank=ep_rank,
)
local_ok = verify_expert_weights_after_shuffle(
expert_weights,
new_indices,
hidden_sizes,
ep_rank,
num_local_experts,
)
local_ok = (
verify_redundant_experts_have_same_weights(
expert_weights,
new_indices,
hidden_sizes,
ep_rank,
world_size,
num_local_experts,
cpu_group=cpu_group,
)
and local_ok
)
assert_verification_synced(
local_ok,
"Async transfer verification failed on at least one rank. "
"See logs for details.",
cpu_group=cpu_group,
)
def _test_rearrange_expert_weights_with_redundancy(
env,
world_size,
num_layers,
num_local_experts,
num_logical_experts,
eplb_communicator: str,
) -> None:
# Initialize model parallel (using tensor parallel as an entrypoint
# to expert parallel)
set_env_vars_and_device(env)
vllm_config = VllmConfig()
vllm_config.parallel_config.tensor_parallel_size = world_size
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
)
ep_group_coordinator = get_tp_group()
ep_group = ep_group_coordinator.device_group
cpu_group = ep_group_coordinator.cpu_group
ep_rank = torch.distributed.get_rank()
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{ep_rank}")
# Test parameters
total_physical_experts = world_size * num_local_experts
hidden_sizes = [32, 64] # Two different weight matrices
# Create old expert indices (with redundancy)
redundancy_config = create_redundancy_config(
num_logical_experts, total_physical_experts
)
old_indices = create_expert_indices_with_redundancy(
num_layers,
num_logical_experts,
total_physical_experts,
redundancy_config,
)
# Create new expert indices (with redundancy)
new_redundancy_config = create_redundancy_config(
num_logical_experts, total_physical_experts
)
new_indices = create_expert_indices_with_redundancy(
num_layers,
num_logical_experts,
total_physical_experts,
new_redundancy_config,
)
# Create expert weights
expert_weights = create_expert_weights(
num_layers, num_local_experts, hidden_sizes, ep_rank, device, old_indices
)
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
communicator = create_eplb_communicator_or_raise(
group_coordinator=ep_group_coordinator,
backend=eplb_communicator,
expert_weights=expert_weights,
expert_buffer=expert_buffer,
)
# Execute weight rearrangement
rearrange_expert_weights_inplace(
old_indices,
new_indices,
expert_weights,
expert_buffer,
ep_group,
communicator,
)
# Verify the rearrangement result
local_ok = verify_expert_weights_after_shuffle(
expert_weights,
new_indices,
hidden_sizes,
ep_rank,
num_local_experts,
)
local_ok = (
verify_redundant_experts_have_same_weights(
expert_weights,
new_indices,
hidden_sizes,
ep_rank,
world_size,
num_local_experts,
cpu_group=cpu_group,
)
and local_ok
)
assert_verification_synced(
local_ok,
"Rearrange verification failed on at least one rank. See logs for details.",
cpu_group=cpu_group,
)
@pytest.mark.parametrize(
"world_size,num_layers,num_local_experts,num_logical_experts",
[
# 2 GPU, 2 experts per GPU
# 3 logical experts, 4 physical experts, 1 redundant experts
(2, 1, 2, 3),
# 2 GPU, 3 experts per GPU
# 4 logical experts, 6 physical experts, 2 redundant experts
(2, 2, 3, 4),
# 2 GPU, 8 experts per GPU
# 16 logical experts, 16 physical experts, 0 redundant experts
(2, 4, 8, 16),
# 4 GPU, 2 experts per GPU
# 6 logical experts, 8 physical experts, 2 redundant experts
(4, 1, 2, 6),
# 4 GPU, 2 experts per GPU
# 5 logical experts, 8 physical experts, 3 redundant experts
(4, 2, 2, 5),
# 4 GPU, 8 experts per GPU
# 16 logical experts, 32 physical experts, 16 redundant experts
(4, 8, 8, 16),
],
)
@pytest.mark.parametrize(
"eplb_communicator", ["torch_nccl", "torch_gloo", "torch_xccl", "pynccl", "nixl"]
)
def test_rearrange_expert_weights_with_redundancy(
world_size,
num_layers,
num_local_experts,
num_logical_experts,
eplb_communicator,
):
"""Test the functionality of rearranging expert weights with redundancy."""
if eplb_communicator == "nixl" and not has_nixl():
pytest.skip("NIXL is not available")
if eplb_communicator == "nixl" and current_platform.is_xpu():
pytest.skip("NIXL does not support XPU")
if eplb_communicator in ("torch_nccl", "pynccl") and not torch.cuda.is_available():
pytest.skip(f"{eplb_communicator} requires CUDA")
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
pytest.skip("torch_xccl requires XPU")
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
distributed_run(
_test_rearrange_expert_weights_with_redundancy,
world_size,
num_layers,
num_local_experts,
num_logical_experts,
eplb_communicator,
)
def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
set_env_vars_and_device(env)
vllm_config = VllmConfig()
vllm_config.parallel_config.tensor_parallel_size = world_size
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
)
ep_group_coordinator = get_tp_group()
ep_group = ep_group_coordinator.device_group
cpu_group = ep_group_coordinator.cpu_group
ep_rank = torch.distributed.get_rank()
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{ep_rank}")
num_layers = 2
num_local_experts = 2
total_physical_experts = world_size * num_local_experts
num_logical_experts = total_physical_experts // 2 # Some redundancy
hidden_sizes = [32, 64]
# Create redundancy configuration
redundancy_config = [2] * num_logical_experts
# Same indices - no change
indices = create_expert_indices_with_redundancy(
num_layers, num_logical_experts, total_physical_experts, redundancy_config
)
expert_weights = create_expert_weights(
num_layers, num_local_experts, hidden_sizes, ep_rank, device, indices
)
# Save original weights
original_weights = []
for layer_weights in expert_weights:
layer_copy = []
for weight in layer_weights:
layer_copy.append(weight.clone())
original_weights.append(layer_copy)
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
communicator = create_eplb_communicator_or_raise(
group_coordinator=ep_group_coordinator,
backend=default_backend,
expert_weights=expert_weights,
expert_buffer=expert_buffer,
)
# Execute rearrangement (should be no change)
rearrange_expert_weights_inplace(
indices,
indices, # Same indices
expert_weights,
expert_buffer,
ep_group,
communicator,
)
# Verify that the weights have not changed
local_ok = True
for layer in range(num_layers):
for weight_idx in range(len(hidden_sizes)):
if not torch.equal(
expert_weights[layer][weight_idx],
original_weights[layer][weight_idx],
):
local_ok = False
print(
"test_rearrange_expert_weights_no_change failed: "
f"layer={layer}, weight_idx={weight_idx}",
flush=True,
)
assert_verification_synced(
local_ok,
"No-change EPLB verification failed on at least one rank.",
cpu_group=cpu_group,
)
@pytest.mark.parametrize(
"world_size,num_layers,num_local_experts,num_logical_experts",
[
(2, 2, 2, 3),
],
)
@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "torch_xccl", "nixl"])
def test_async_transfer_layer_without_mtp(
world_size: int,
num_layers: int,
num_local_experts: int,
num_logical_experts: int,
eplb_communicator: str,
):
"""Exercise async EPLB transfer path without MTP/spec decode."""
if eplb_communicator == "nixl" and not has_nixl():
pytest.skip("NIXL is not available")
if eplb_communicator == "nixl" and current_platform.is_xpu():
pytest.skip("NIXL does not support XPU")
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
pytest.skip("torch_xccl requires XPU")
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
distributed_run(
_test_async_transfer_layer_without_mtp_worker,
world_size,
num_layers,
num_local_experts,
num_logical_experts,
eplb_communicator,
)
@pytest.mark.parametrize("world_size", [2, 4])
def test_rearrange_expert_weights_no_change(world_size):
"""Test that when the indices do not change, the weights should remain
unchanged.
"""
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
distributed_run(
_test_rearrange_expert_weights_no_change,
world_size,
)
def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
set_env_vars_and_device(env)
vllm_config = VllmConfig()
vllm_config.parallel_config.tensor_parallel_size = world_size
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
)
ep_group_coordinator = get_tp_group()
ep_group = ep_group_coordinator.device_group
cpu_group = ep_group_coordinator.cpu_group
ep_rank = torch.distributed.get_rank()
accelerator_type = torch.accelerator.current_accelerator().type
device = torch.device(f"{accelerator_type}:{ep_rank}")
num_layers = 1
num_local_experts = 2
total_physical_experts = world_size * num_local_experts
num_logical_experts = total_physical_experts // 2
hidden_sizes = [32]
# Create different index distributions
old_redundancy = create_redundancy_config(
num_logical_experts, total_physical_experts
)
new_redundancy = create_redundancy_config(
num_logical_experts, total_physical_experts
)
old_indices = create_expert_indices_with_redundancy(
num_layers, num_logical_experts, total_physical_experts, old_redundancy
)
new_indices = create_expert_indices_with_redundancy(
num_layers, num_logical_experts, total_physical_experts, new_redundancy
)
expert_weights = create_expert_weights(
num_layers, num_local_experts, hidden_sizes, ep_rank, device, old_indices
)
# Save original weights
original_weights = []
for layer_weights in expert_weights:
layer_copy = []
for weight in layer_weights:
layer_copy.append(weight.clone())
original_weights.append(layer_copy)
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
communicator = create_eplb_communicator_or_raise(
group_coordinator=ep_group_coordinator,
backend=default_backend,
expert_weights=expert_weights,
expert_buffer=expert_buffer,
)
# Execute profile mode rearrangement
rearrange_expert_weights_inplace(
old_indices,
new_indices,
expert_weights,
expert_buffer,
ep_group,
communicator,
is_profile=True,
)
# In profile mode, the weights should remain unchanged
local_ok = True
for layer in range(num_layers):
for weight_idx in range(len(hidden_sizes)):
if not torch.equal(
expert_weights[layer][weight_idx],
original_weights[layer][weight_idx],
):
local_ok = False
print(
"test_rearrange_expert_weights_profile_mode failed: "
f"layer={layer}, weight_idx={weight_idx}",
flush=True,
)
assert_verification_synced(
local_ok,
"Profile-mode EPLB verification failed on at least one rank.",
cpu_group=cpu_group,
)
@pytest.mark.parametrize("world_size", [2, 4])
def test_rearrange_expert_weights_profile_mode(world_size):
"""Test profile mode (should not copy actual weights)"""
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
distributed_run(
_test_rearrange_expert_weights_profile_mode,
world_size,
)
def _test_eplb_communicator_pipeline_parallel(env, world_size, backend):
set_env_vars_and_device(env)
config = VllmConfig()
config.parallel_config.tensor_parallel_size = 2
config.parallel_config.pipeline_parallel_size = world_size // 2
with set_current_vllm_config(config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=2,
pipeline_model_parallel_size=world_size // 2,
)
ep_group = get_tp_group()
peer_rank = 1 - ep_group.rank_in_group
rank = torch.distributed.get_rank()
send_tensor = torch.full((16,), rank, device="cuda", dtype=torch.int32)
recv_tensor = torch.empty_like(send_tensor)
communicator = create_eplb_communicator_or_raise(
group_coordinator=ep_group,
backend=backend,
expert_weights=[[send_tensor]],
expert_buffer=[recv_tensor],
)
communicator.add_send([send_tensor], peer_rank, expert_id=0)
communicator.add_recv([recv_tensor], peer_rank, expert_id=0)
communicator.execute()
expected = torch.full_like(recv_tensor, ep_group.ranks[peer_rank])
torch.testing.assert_close(recv_tensor, expected)
@pytest.mark.parametrize("world_size", [2, 4])
@pytest.mark.parametrize("backend", ["torch_nccl", "torch_gloo"])
def test_eplb_communicator_pipeline_parallel(world_size, backend):
"""Transfer weights within each PP stage, including nonzero global ranks."""
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
distributed_run(_test_eplb_communicator_pipeline_parallel, world_size, backend)
@@ -1,298 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Test that the interaction between EPLB and FusedMoEFactory Layer is okay
from dataclasses import dataclass
import pytest
import torch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
from vllm.distributed.eplb.rebalance_execute import rearrange_expert_weights_inplace
from vllm.distributed.parallel_state import (
ensure_model_parallel_initialized,
get_eplb_group,
get_tp_group,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
from vllm.platforms import current_platform
from .eplb_utils import distributed_run, set_env_vars_and_device
@dataclass
class TestConfig:
num_layers: int
num_experts: int
num_local_experts: int
num_topk: int
hidden_size: int
intermediate_size: int
weight_dtype: torch.dtype
weight_scale_dtype: torch.dtype | None
column_major_scales: bool
def make_expert_weights(
layer_idx: int,
global_expert_idx: int,
global_num_experts: int,
tensor_shape: tuple[int, ...],
tensor_dtype: torch.dtype,
tensor_device: torch.device,
is_column_major: bool,
) -> torch.Tensor:
assert len(tensor_shape) == 2
if is_column_major:
tensor_shape = (tensor_shape[1], tensor_shape[0])
x = torch.empty(tensor_shape, dtype=tensor_dtype, device=tensor_device)
value_offset = (layer_idx * global_num_experts + global_expert_idx) * x.numel()
x.view(-1).copy_(
torch.arange(
value_offset,
value_offset + x.numel(),
dtype=tensor_dtype,
device=tensor_device,
)
)
if is_column_major:
x = torch.transpose(x, 1, 0)
assert not x.is_contiguous()
return x
def make_fused_moe_layer(
rank: int,
layer_idx: int,
test_config: TestConfig,
) -> MoERunner:
fml = FusedMoEFactory(
num_experts=test_config.num_experts,
top_k=test_config.num_topk,
hidden_size=test_config.hidden_size,
intermediate_size=test_config.intermediate_size,
prefix=f"dummy_layer_{layer_idx}",
activation="silu",
params_dtype=test_config.weight_dtype,
)
re = fml.routed_experts
device = torch.device(f"{torch.accelerator.current_accelerator().type}:{rank}")
from functools import partial
_make_expert_weights = partial(
make_expert_weights,
layer_idx=layer_idx,
global_num_experts=test_config.num_experts,
tensor_device=device,
)
assert isinstance(re.w13_weight.data, torch.Tensor)
assert isinstance(re.w2_weight.data, torch.Tensor)
re.w13_weight.data = re.w13_weight.data.to(device=device)
re.w2_weight.data = re.w2_weight.data.to(device=device)
w13_weight = re.w13_weight.data
w2_weight = re.w2_weight.data
assert w13_weight.size(0) == test_config.num_local_experts
for i in range(test_config.num_local_experts):
g_i = rank * test_config.num_local_experts + i
w13_weight_e = w13_weight[i]
w2_weight_e = w2_weight[i]
w13_weight_e.copy_(
_make_expert_weights(
global_expert_idx=g_i,
tensor_shape=w13_weight_e.shape,
tensor_dtype=w13_weight_e.dtype,
is_column_major=False,
)
)
w2_weight_e.copy_(
_make_expert_weights(
global_expert_idx=g_i,
tensor_shape=w2_weight_e.shape,
tensor_dtype=w2_weight_e.dtype,
is_column_major=False,
)
)
block_size = 16
def block_quant_scales_shape(
shape: tuple[int, ...], is_column_major: bool
) -> tuple[int, ...]:
assert len(shape) == 3
if not is_column_major:
return (shape[0], shape[1] // block_size, shape[2] // block_size)
else:
return (shape[0], shape[2] // block_size, shape[1] // block_size)
is_column_major = test_config.column_major_scales
w13_weight_scale_inv = torch.empty(
block_quant_scales_shape(w13_weight.shape, is_column_major),
dtype=test_config.weight_dtype,
device=device,
)
w2_weight_scale_inv = torch.empty(
block_quant_scales_shape(w2_weight.shape, is_column_major),
dtype=test_config.weight_dtype,
device=device,
)
for i in range(test_config.num_local_experts):
g_i = rank * test_config.num_local_experts + i
w13_s_e = w13_weight_scale_inv[i]
w2_s_e = w2_weight_scale_inv[i]
w13_s_e.copy_(
_make_expert_weights(
global_expert_idx=g_i,
tensor_shape=w13_s_e.shape,
tensor_dtype=w13_s_e.dtype,
# Fill data in row-major and then
# transpose if test_config requires col-major.
is_column_major=False,
)
)
w2_s_e.copy_(
_make_expert_weights(
global_expert_idx=g_i,
tensor_shape=w2_s_e.shape,
tensor_dtype=w2_s_e.dtype,
is_column_major=False,
)
)
if is_column_major:
w13_weight_scale_inv = torch.transpose(w13_weight_scale_inv, 1, 2)
w2_weight_scale_inv = torch.transpose(w2_weight_scale_inv, 1, 2)
assert not w13_weight_scale_inv.is_contiguous()
assert not w2_weight_scale_inv.is_contiguous()
# Add scales to the parameter list
re.w13_weight_scale_inv = torch.nn.Parameter(
w13_weight_scale_inv, requires_grad=False
)
re.w2_weight_scale_inv = torch.nn.Parameter(
w2_weight_scale_inv, requires_grad=False
)
return fml
def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
# Initialize model parallel (using tensor parallel as an entrypoint
# to expert parallel)
set_env_vars_and_device(env)
vllm_config = VllmConfig()
vllm_config.parallel_config.tensor_parallel_size = world_size
vllm_config.parallel_config.enable_expert_parallel = True
vllm_config.parallel_config.enable_eplb = True
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
)
ep_group = get_tp_group().device_group
ep_rank = torch.distributed.get_rank()
fml_layers = [
make_fused_moe_layer(ep_rank, layer_idx, test_config)
for layer_idx in range(test_config.num_layers)
]
rank_expert_weights = [fml.get_expert_weights() for fml in fml_layers]
indices = torch.zeros(
test_config.num_layers, test_config.num_experts, dtype=torch.long
)
for lidx in range(test_config.num_layers):
indices[lidx] = torch.Tensor(range(test_config.num_experts))
shuffled_indices = torch.zeros_like(indices)
for lidx in range(test_config.num_layers):
shuffled_indices[lidx] = torch.randperm(test_config.num_experts)
expert_buffer = [torch.empty_like(w) for w in rank_expert_weights[0]]
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
communicator = create_eplb_communicator(
group_coordinator=get_eplb_group(),
backend=default_backend,
expert_weights=rank_expert_weights,
expert_buffer=expert_buffer,
)
rearrange_expert_weights_inplace(
indices,
shuffled_indices,
rank_expert_weights,
expert_buffer,
ep_group,
communicator,
)
num_local_experts = test_config.num_local_experts
num_global_experts = test_config.num_experts
for lidx, fml in enumerate(fml_layers):
for name, w in fml.named_parameters():
for e in range(num_local_experts):
g_e = shuffled_indices[lidx][ep_rank * num_local_experts + e]
ref = make_expert_weights(
layer_idx=lidx,
global_expert_idx=int(g_e.item()),
global_num_experts=num_global_experts,
tensor_shape=w[e].shape,
tensor_dtype=w[e].dtype,
tensor_device=w[e].device,
is_column_major=not w[e].is_contiguous(),
)
assert w[e].shape == ref.shape and w[e].stride() == ref.stride(), (
f"w[{e}] {w[e].size()} {w[e].stride()} vs "
f"ref {ref.size()} {ref.stride()}"
)
torch.testing.assert_close(w[e], ref)
@pytest.mark.parametrize("world_size", [2])
@pytest.mark.parametrize("num_layers", [4])
@pytest.mark.parametrize("num_experts", [16])
@pytest.mark.parametrize("hidden_size", [256])
@pytest.mark.parametrize("intermediate_size", [256])
@pytest.mark.parametrize("column_major_scales", [True, False])
def test_eplb_fml(
world_size: int,
num_layers: int,
num_experts: int,
hidden_size: int,
intermediate_size: int,
column_major_scales: bool,
):
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
num_local_experts = num_experts // world_size
num_topk = 4
# The dtypes are fine as we are essentially just checking data-copies
weight_dtype = torch.bfloat16
weight_scale_dtype = torch.bfloat16
test_config = TestConfig(
num_layers=num_layers,
num_experts=num_experts,
num_local_experts=num_local_experts,
num_topk=num_topk,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
weight_dtype=weight_dtype,
weight_scale_dtype=weight_scale_dtype,
column_major_scales=column_major_scales,
)
distributed_run(
_test_eplb_fml,
world_size,
test_config,
)
@@ -1,295 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Test that the interaction between EPLB and MoERunner Layer is okay for DP w/ NVFP4
from dataclasses import dataclass
import pytest
import torch
from tests.kernels.moe.utils import make_test_quant_config
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.distributed.eplb.rebalance_execute import rearrange_expert_weights_inplace
from vllm.distributed.parallel_state import (
ensure_model_parallel_initialized,
get_dp_group,
get_eplb_group,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory, MoERunner
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptNvFp4Config,
ModelOptNvFp4FusedMoE,
)
from vllm.platforms import current_platform
from .eplb_utils import distributed_run, set_env_vars_and_device
@dataclass
class TestConfig:
num_layers: int
num_experts: int
num_local_experts: int
num_topk: int
hidden_size: int
intermediate_size: int
num_tokens: int
moe_backend: str
def make_fused_moe_layer(
rank: int,
layer_idx: int,
test_config: TestConfig,
) -> MoERunner:
quant_config = None
device = torch.device(f"cuda:{rank}")
quant_config = ModelOptNvFp4Config(
is_checkpoint_nvfp4_serialized=True,
kv_cache_quant_algo=None,
exclude_modules=[],
)
fml = FusedMoEFactory(
num_experts=test_config.num_experts,
top_k=test_config.num_topk,
hidden_size=test_config.hidden_size,
intermediate_size=test_config.intermediate_size,
prefix=f"dummy_layer_{layer_idx}",
activation="silu",
params_dtype=torch.bfloat16,
quant_config=quant_config,
)
nvfp4_fused_moe = ModelOptNvFp4FusedMoE(quant_config, fml)
nvfp4_fused_moe.create_weights(
fml,
test_config.num_local_experts,
test_config.hidden_size,
test_config.intermediate_size,
params_dtype=torch.uint8,
global_num_experts=test_config.num_experts,
)
fml = fml.to(device)
re = fml.routed_experts
w1_q, w2_q, quant_config = make_test_quant_config(
test_config.num_local_experts,
test_config.intermediate_size,
test_config.hidden_size,
in_dtype=torch.bfloat16,
quant_dtype="nvfp4",
block_shape=None,
per_act_token_quant=False,
)
re.w13_weight.data = w1_q
re.w2_weight.data = w2_q
re.w2_input_scale.data = torch.randn_like(re.w2_input_scale.data) / 5
re.w13_input_scale.data = torch.randn_like(re.w13_input_scale.data) / 5
re.w2_weight_scale_2.data = torch.randn_like(re.w2_weight_scale_2.data) / 5
re.w13_weight_scale_2.data = torch.randn_like(re.w13_weight_scale_2.data) / 5
re.w2_weight_scale.data = (
torch.randn(re.w2_weight_scale.data.shape, device=device) / 5
).to(re.w2_weight_scale.data.dtype)
re.w13_weight_scale.data = (
torch.randn(re.w13_weight_scale.data.shape, device=device) / 5
).to(re.w13_weight_scale.data.dtype)
nvfp4_fused_moe.process_weights_after_loading(re)
fml.maybe_init_modular_kernel()
return fml
def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
set_env_vars_and_device(env)
vllm_config = VllmConfig()
vllm_config.parallel_config.data_parallel_size = world_size
vllm_config.parallel_config.enable_expert_parallel = True
vllm_config.kernel_config.moe_backend = test_config.moe_backend
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=1, pipeline_model_parallel_size=1
)
ep_group = get_dp_group().cpu_group
ep_rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{ep_rank}")
fml_layers = [
make_fused_moe_layer(ep_rank, layer_idx, test_config).to(device)
for layer_idx in range(test_config.num_layers)
]
rank_expert_weights = [fml.get_expert_weights() for fml in fml_layers]
hidden_states = []
router_logits = []
for layer_idx in range(test_config.num_layers):
hidden_states.append(
torch.randn(
(test_config.num_tokens, test_config.hidden_size),
dtype=torch.bfloat16,
device=device,
)
)
router_logits.append(
torch.randn(
(test_config.num_tokens, test_config.num_experts),
dtype=torch.bfloat16,
device=device,
)
)
out_before_shuffle = []
with set_forward_context(
{},
num_tokens=test_config.num_tokens,
num_tokens_across_dp=torch.tensor(
[test_config.num_tokens] * world_size, device="cpu", dtype=torch.int
),
vllm_config=vllm_config,
):
for lidx, fml in enumerate(fml_layers):
out_before_shuffle.append(
fml(hidden_states[lidx].clone(), router_logits[lidx].clone())
)
indices = torch.zeros(
test_config.num_layers, test_config.num_experts, dtype=torch.long
)
for lidx in range(test_config.num_layers):
indices[lidx] = torch.Tensor(range(test_config.num_experts))
shuffled_indices = torch.zeros_like(indices)
for lidx in range(test_config.num_layers):
shuffled_indices[lidx] = torch.randperm(test_config.num_experts)
expert_buffer = [torch.empty_like(w) for w in rank_expert_weights[0]]
communicator = create_eplb_communicator(
group_coordinator=get_eplb_group(),
backend="torch_nccl",
expert_weights=rank_expert_weights,
expert_buffer=expert_buffer,
)
rearrange_expert_weights_inplace(
indices,
shuffled_indices,
rank_expert_weights,
expert_buffer,
ep_group,
communicator,
)
num_global_experts = test_config.num_experts
logical_to_physical_map_list = []
for lidx, fml in enumerate(fml_layers):
physical_to_logical_map = shuffled_indices[lidx].to(device)
logical_to_physical_map = torch.empty(
(num_global_experts,), dtype=torch.int32, device=device
)
logical_to_physical_map[physical_to_logical_map] = torch.arange(
0, num_global_experts, dtype=torch.int32, device=device
)
logical_to_physical_map_list.append(
logical_to_physical_map.reshape(num_global_experts, 1)
)
logical_to_physical_map = torch.stack(logical_to_physical_map_list)
for lidx, fml in enumerate(fml_layers):
logical_replica_count = torch.ones(
(test_config.num_layers, num_global_experts),
dtype=torch.int32,
device=device,
)
fml.eplb_state = EplbLayerState()
fml.set_eplb_state(
lidx,
torch.zeros(
(test_config.num_layers, num_global_experts),
dtype=torch.int32,
device=device,
),
logical_to_physical_map,
logical_replica_count,
)
fml.router.eplb_state.should_record_tensor = torch.ones(
(), dtype=torch.bool, device=device
)
fml.router.eplb_state.num_unpadded_tokens_tensors = [
torch.tensor(0, dtype=torch.int32, device=device)
]
out_after_shuffle = []
with set_forward_context(
{},
num_tokens=test_config.num_tokens,
num_tokens_across_dp=torch.tensor(
[test_config.num_tokens] * world_size, device="cpu", dtype=torch.int
),
vllm_config=vllm_config,
):
for lidx, fml in enumerate(fml_layers):
out_after_shuffle.append(
fml(hidden_states[lidx].clone(), router_logits[lidx].clone())
)
for lidx in range(test_config.num_layers):
torch.testing.assert_close(
out_before_shuffle[lidx], out_after_shuffle[lidx], atol=1e-1, rtol=1e-1
)
@pytest.mark.parametrize("world_size", [2, 4])
@pytest.mark.parametrize("num_layers", [8])
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("hidden_size", [256])
@pytest.mark.parametrize("intermediate_size", [256])
@pytest.mark.parametrize("num_tokens", [256])
@pytest.mark.parametrize("moe_backend", ["flashinfer_trtllm", "flashinfer_cutlass"])
def test_eplb_fml(
world_size: int,
num_layers: int,
num_experts: int,
hidden_size: int,
intermediate_size: int,
num_tokens: int,
moe_backend: str,
):
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
if current_platform.is_xpu():
pytest.skip("NVFP4 quantization and flashinfer backends are CUDA-only")
num_local_experts = num_experts // world_size
num_topk = 4
test_config = TestConfig(
num_layers=num_layers,
num_experts=num_experts,
num_local_experts=num_local_experts,
num_topk=num_topk,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
num_tokens=num_tokens,
moe_backend=moe_backend,
)
distributed_run(
_test_eplb_fml,
world_size,
test_config,
)
@@ -1,299 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""EPLB rearrangement consistency tests for quant-method derived state.
EPLB rearranges every registered Parameter of a RoutedExperts in place,
sliced along dim 0 (see RoutedExperts.get_expert_weights). Quant methods
must therefore register all derived per-expert tensors as Parameters and
alias the same storage in their FusedMoEQuantConfig, so the kernels observe
rearranged values with no extra bookkeeping. These tests verify that
contract: simulating a rearrangement on the registered Parameters must be
indistinguishable from loading a checkpoint with permuted experts.
"""
import pytest
import torch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_w4a4_nvfp4 import ( # noqa: E501
CompressedTensorsW4A4Nvfp4MoEMethod,
)
from vllm.platforms import current_platform
EPLB_NVFP4_BACKENDS = ["flashinfer_cutedsl", "flashinfer_trtllm"]
NUM_EXPERTS = 8
HIDDEN_SIZE = 128
INTERMEDIATE_SIZE = 256
EXPERT_PERMUTATION = [3, 0, 5, 1, 7, 2, 6, 4]
QUANT_CONFIG_TENSORS = (
"w1_scale",
"w2_scale",
"g1_alphas",
"g2_alphas",
"a1_gscale",
"a2_gscale",
)
class _RoutedExpertsStub(torch.nn.Module):
"""Minimal ``RoutedExperts`` stand-in for NVFP4 weight processing."""
def __init__(self, moe_config: FusedMoEConfig) -> None:
super().__init__()
self.moe_config = moe_config
self.activation = moe_config.activation
def _expert_routing_tables(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
return None
def _make_moe_config(backend: str) -> FusedMoEConfig:
parallel_config = FusedMoEParallelConfig(
tp_size=1,
pcp_size=1,
dp_size=1,
ep_size=1,
tp_rank=0,
pcp_rank=0,
dp_rank=0,
ep_rank=0,
sp_size=1,
use_ep=True,
all2all_backend="allgather_reducescatter",
enable_eplb=True,
)
return FusedMoEConfig(
num_experts=NUM_EXPERTS,
experts_per_token=2,
hidden_dim=HIDDEN_SIZE,
intermediate_size=INTERMEDIATE_SIZE,
num_local_experts=NUM_EXPERTS,
num_logical_experts=NUM_EXPERTS,
activation=MoEActivation.SILU,
device="cuda",
routing_method=RoutingMethodType.TopK,
moe_parallel_config=parallel_config,
in_dtype=torch.bfloat16,
moe_backend=backend,
)
def _make_raw_weights(
device: torch.device, generator: torch.Generator
) -> dict[str, torch.Tensor]:
"""Random raw NVFP4 checkpoint tensors, indexable per-expert on dim 0."""
e, h, i = NUM_EXPERTS, HIDDEN_SIZE, INTERMEDIATE_SIZE
def packed(*shape: int) -> torch.Tensor:
return torch.randint(
0, 256, shape, dtype=torch.uint8, device=device, generator=generator
)
def block_scale(*shape: int) -> torch.Tensor:
return torch.randint(
0, 128, shape, dtype=torch.uint8, device=device, generator=generator
).view(torch.float8_e4m3fn)
def global_scale(*shape: int) -> torch.Tensor:
return torch.rand(shape, device=device, generator=generator) + 0.5
return {
"w13_weight_packed": packed(e, 2 * i, h // 2),
"w2_weight_packed": packed(e, h, i // 2),
"w13_weight_scale": block_scale(e, 2 * i, h // 16),
"w2_weight_scale": block_scale(e, h, i // 16),
"w13_weight_global_scale": global_scale(e, 2),
"w2_weight_global_scale": global_scale(e),
"w13_input_global_scale": global_scale(e, 2),
"w2_input_global_scale": global_scale(e),
}
def _build_processed_layer(
backend: str, raw: dict[str, torch.Tensor], device: torch.device
) -> tuple[CompressedTensorsW4A4Nvfp4MoEMethod, _RoutedExpertsStub]:
"""Create the method + layer, load ``raw`` and run the real
process_weights_after_loading (which builds the flashinfer kernel)."""
moe_config = _make_moe_config(backend)
method = CompressedTensorsW4A4Nvfp4MoEMethod(moe_config, "layer.0", use_a16=False)
layer = _RoutedExpertsStub(moe_config).to(device)
method.create_weights(
layer,
num_experts=NUM_EXPERTS,
hidden_size=HIDDEN_SIZE,
intermediate_size_per_partition=INTERMEDIATE_SIZE,
params_dtype=torch.bfloat16,
weight_loader=lambda *args, **kwargs: None,
)
layer = layer.to(device)
with torch.no_grad():
for name, value in raw.items():
getattr(layer, name).copy_(value)
method.process_weights_after_loading(layer)
return method, layer
def _assert_tensors_equal(
actual: torch.Tensor, expected: torch.Tensor, name: str
) -> None:
if actual.dtype == torch.float8_e4m3fn:
assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8)), name
elif actual.dtype == torch.uint8:
assert torch.equal(actual, expected), name
else:
torch.testing.assert_close(actual, expected, msg=lambda m: f"{name}: {m}")
def _simulate_eplb_rearrangement(layer: torch.nn.Module, perm: torch.Tensor) -> None:
"""Permute experts of every registered Parameter in place, the way
EPLB's in-place rearrangement moves expert slices along dim 0."""
with torch.no_grad():
for _, param in layer.named_parameters():
param.copy_(param[perm])
def _ensure_world1_distributed() -> None:
"""The enable_eplb config makes weight processing all-reduce activation
scale amaxes over the EP group, so a (single-rank) EP group must exist."""
from vllm.distributed.parallel_state import (
ensure_model_parallel_initialized,
init_distributed_environment,
)
if not torch.distributed.is_initialized():
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method="tcp://127.0.0.1:0",
local_rank=0,
)
ensure_model_parallel_initialized(1, 1)
@pytest.mark.parametrize("backend", EPLB_NVFP4_BACKENDS)
def test_nvfp4_eplb_rearrangement_matches_reload(backend: str) -> None:
if not (
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
):
pytest.skip("NVFP4 CuteDSL/TRTLLM MoE backends require Blackwell (SM100).")
device = torch.device("cuda:0")
torch.accelerator.set_device_index(device)
perm = torch.tensor(EXPERT_PERMUTATION, device=device)
with set_current_vllm_config(VllmConfig()):
_ensure_world1_distributed()
generator = torch.Generator(device=device).manual_seed(1234)
raw = _make_raw_weights(device, generator)
raw_permuted = {name: value[perm].contiguous() for name, value in raw.items()}
try:
method, layer = _build_processed_layer(backend, raw, device)
except ValueError as exc:
pytest.skip(f"{backend} NVFP4 MoE backend unavailable: {exc}")
ref_method, ref_layer = _build_processed_layer(backend, raw_permuted, device)
# The EPLB contract: every registered Parameter must be an
# expert-major contiguous tensor so get_expert_weights can view it
# as (E, -1) and rearrange expert slices in place.
for name, param in layer.named_parameters():
assert param.is_contiguous(), f"{name} is not contiguous"
assert param.shape[0] == NUM_EXPERTS, (
f"{name} is not expert-major: {tuple(param.shape)}"
)
# Derived per-expert scales must live in registered Parameters that
# the quant config aliases; anything else goes stale on rearrangement.
quant_config = method.moe_quant_config
assert quant_config is not None
params = dict(layer.named_parameters())
assert quant_config.g1_alphas.data_ptr() == (
params["w13_weight_scale_2"].data_ptr()
)
assert quant_config.g2_alphas.data_ptr() == (
params["w2_weight_scale_2"].data_ptr()
)
assert quant_config.w1_scale.data_ptr() == (
params["w13_weight_scale"].data_ptr()
)
assert quant_config.w2_scale.data_ptr() == params["w2_weight_scale"].data_ptr()
_simulate_eplb_rearrangement(layer, perm)
# After rearrangement, all registered state must equal what a fresh
# load of the permuted experts produces.
ref_params = dict(ref_layer.named_parameters())
assert params.keys() == ref_params.keys()
for name in params:
_assert_tensors_equal(params[name], ref_params[name], name)
# And so must the kernel-visible quant config tensors (the CuteDSL
# MMA scale views alias the registered Parameters' storage).
ref_quant_config = ref_method.moe_quant_config
assert ref_quant_config is not None
for name in QUANT_CONFIG_TENSORS:
actual = getattr(quant_config, name)
expected = getattr(ref_quant_config, name)
assert (actual is None) == (expected is None), name
if actual is not None:
_assert_tensors_equal(actual, expected, f"quant_config.{name}")
def test_fp8_per_tensor_alphas_registered_and_aliased() -> None:
"""CPU-only: the fp8 per-tensor oracle must register the fused
(w_scale * a_scale) products as layer Parameters aliased by the quant
config, so EPLB rearrangement keeps kernels consistent."""
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
make_fp8_moe_quant_config,
)
e = NUM_EXPERTS
generator = torch.Generator().manual_seed(1234)
def scales() -> tuple[torch.Tensor, torch.Tensor]:
w_scale = torch.rand((e,), generator=generator) + 0.5
a_scale = torch.rand((), generator=generator) + 0.5
return w_scale, a_scale.expand(e).contiguous()
w1_scale, a1_scale = scales()
w2_scale, a2_scale = scales()
layer = torch.nn.Module()
quant_config = make_fp8_moe_quant_config(
fp8_backend=Fp8MoeBackend.FLASHINFER_CUTLASS,
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=a1_scale,
a2_scale=a2_scale,
layer=layer,
)
params = dict(layer.named_parameters())
assert quant_config.g1_alphas.data_ptr() == params["g1_alphas"].data_ptr()
assert quant_config.g2_alphas.data_ptr() == params["g2_alphas"].data_ptr()
perm = torch.tensor(EXPERT_PERMUTATION)
ref_config = make_fp8_moe_quant_config(
fp8_backend=Fp8MoeBackend.FLASHINFER_CUTLASS,
w1_scale=w1_scale[perm].contiguous(),
w2_scale=w2_scale[perm].contiguous(),
a1_scale=a1_scale,
a2_scale=a2_scale,
layer=None,
)
_simulate_eplb_rearrangement(layer, perm)
torch.testing.assert_close(quant_config.g1_alphas, ref_config.g1_alphas)
torch.testing.assert_close(quant_config.g2_alphas, ref_config.g2_alphas)
-408
View File
@@ -1,408 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
import time
import msgspec
import pytest
from vllm.distributed.kv_events import (
EventBatch,
EventPublisherFactory,
NullEventPublisher,
)
DP_RANK = 0
class EventSample(
msgspec.Struct,
tag=True, # type: ignore
array_like=True, # type: ignore
):
"""Test event for publisher testing."""
id: int
value: str
class SampleBatch(EventBatch):
"""Test event batch for publisher testing."""
events: list[EventSample]
def create_test_events(count: int) -> SampleBatch:
"""Create a batch of test events."""
events = [EventSample(id=i, value=f"test-{i}") for i in range(count)]
return SampleBatch(ts=time.time(), events=events)
def test_basic_publishing(publisher, subscriber):
"""Test basic event publishing works."""
test_batch = create_test_events(5)
publisher.publish(test_batch)
result = subscriber.receive_one(timeout=1000)
assert result is not None, "No message received"
seq, received = result
assert seq == 0, "Sequence number mismatch"
assert received.ts == pytest.approx(test_batch.ts, abs=0.1), "Timestamp mismatch"
assert len(received.events) == len(test_batch.events), "Number of events mismatch"
for i, event in enumerate(received.events):
assert event.id == i, "Event id mismatch"
assert event.value == f"test-{i}", "Event value mismatch"
def test_multiple_events(publisher, subscriber):
"""Test publishing and receiving multiple event batches."""
for _ in range(10):
batch = create_test_events(2)
publisher.publish(batch)
received = []
for _ in range(10):
data = subscriber.receive_one(timeout=100)
if data:
received.append(data)
assert len(received) == 10, "Number of messages mismatch"
seqs = [seq for seq, _ in received]
assert seqs == list(range(10)), "Sequence numbers mismatch"
def test_replay_mechanism(publisher, subscriber):
"""Test the replay mechanism works correctly."""
for _ in range(19):
batch = create_test_events(1)
publisher.publish(batch)
# Drain live events to ensure publisher has buffered them.
for _ in range(19):
assert subscriber.receive_one(timeout=1000) is not None
subscriber.request_replay(10)
replayed = subscriber.receive_replay()
assert len(replayed) == 9, (
f"Expected 9 replayed messages (seq 10-18), got {len(replayed)}"
)
seqs = [seq for seq, _ in replayed]
assert seqs == list(range(10, 19)), "Replayed sequences should be 10-18"
def test_replay_includes_topic(publisher, subscriber, publisher_config):
"""Test that replay responses include the topic, matching PUB format."""
for _ in range(5):
publisher.publish(create_test_events(1))
# Drain live events to ensure publisher has processed them.
for _ in range(5):
assert subscriber.receive_one(timeout=1000) is not None
subscriber.request_replay(0)
# receive_replay unpacks (topic, seq, payload) and asserts
# topic == publisher topic for each message.
replayed = subscriber.receive_replay()
assert len(replayed) == 5, f"Expected 5 replayed messages, got {len(replayed)}"
seqs = [seq for seq, _ in replayed]
assert seqs == list(range(5)), "Replayed sequences should be 0-4"
def test_buffer_limit(publisher, subscriber, publisher_config):
"""Test buffer limit behavior."""
buffer_size = publisher_config.buffer_steps
# Publish more events than the buffer can hold
for i in range(buffer_size + 10):
batch = create_test_events(1)
publisher.publish(batch)
time.sleep(0.5) # Need publisher to process above requests
subscriber.request_replay(0)
replayed = subscriber.receive_replay()
assert len(replayed) == buffer_size, (
f"Expected {buffer_size} replayed messages, got {len(replayed)}"
)
seqs = [seq for seq, _ in replayed]
assert seqs == list(range(10, buffer_size + 10)), (
"Should replay seq 11 through buffer_size+10"
)
def test_topic_filtering(publisher_config):
"""Test that a subscriber only receives messages matching its topic filter."""
publisher_config.replay_endpoint = None
publisher_config.topic = "foo"
pub = EventPublisherFactory.create(publisher_config, DP_RANK)
from .conftest import MockSubscriber
sub_foo = MockSubscriber(publisher_config.endpoint, None, "foo")
sub_bar = MockSubscriber(publisher_config.endpoint, None, "bar")
try:
time.sleep(0.1)
for _ in range(3):
pub.publish(create_test_events(1))
foo_received = [sub_foo.receive_one(timeout=200) for _ in range(3)]
assert all(msg is not None for msg in foo_received), (
"Subscriber with matching topic should receive messages"
)
bar_received = [sub_bar.receive_one(timeout=200) for _ in range(3)]
assert all(msg is None for msg in bar_received), (
"Subscriber with non-matching topic should receive no messages"
)
finally:
pub.shutdown()
sub_foo.close()
sub_bar.close()
def test_high_volume(publisher, subscriber):
"""Test publishing and receiving a high volume of events."""
num_batches = 10_000
events_per_batch = 100
# Publish events in a separate thread to not block
def publish_events():
for i in range(num_batches):
batch = create_test_events(events_per_batch)
publisher.publish(batch)
# Small delay to avoid overwhelming
if i % 100 == 0:
time.sleep(0.01)
received: list[tuple[int, SampleBatch]] = []
publisher_thread = threading.Thread(target=publish_events)
publisher_thread.start()
start_time = time.time()
while len(received) < num_batches:
if time.time() - start_time > 10: # Timeout after 10 seconds
break
result = subscriber.receive_one(timeout=100)
if result:
received.append(result)
publisher_thread.join()
assert len(received) >= num_batches * 0.9, "We should have received most messages"
seqs = [seq for seq, _ in received]
assert sorted(seqs) == seqs, "Sequence numbers should be in order"
def test_null_publisher():
"""Test that NullEventPublisher can be used without errors."""
publisher = NullEventPublisher(DP_RANK)
# This should not raise any errors
batch = create_test_events(5)
publisher.publish(batch)
publisher.shutdown()
def test_data_parallel_rank_tagging(publisher_config):
"""Test that events are properly tagged with their data parallel rank."""
publisher_config.topic = "foo"
pub_0 = EventPublisherFactory.create(publisher_config, DP_RANK)
pub_1 = EventPublisherFactory.create(publisher_config, DP_RANK + 1)
# Hardcode the expected endpoints based on port offsetting behavior
# Both ranks get offsets according to _offset_endpoint_port function
base_endpoint = publisher_config.endpoint
if "tcp://" in base_endpoint:
# For TCP endpoints: tcp://localhost:5557 -> tcp://localhost:5557, tcp://localhost:5558
expected_endpoint_0 = base_endpoint # rank 0 gets port + 0 = same port
expected_endpoint_1 = base_endpoint.replace(
":5557", ":5558"
) # rank 1 gets port + 1
else:
# For inproc endpoints: inproc://test -> inproc://test_dp0, inproc://test_dp1
expected_endpoint_0 = base_endpoint # rank 0 gets base
expected_endpoint_1 = base_endpoint + "_dp1" # rank 1 gets _dp1
from .conftest import MockSubscriber
sub_0 = MockSubscriber(expected_endpoint_0, None, publisher_config.topic)
sub_1 = MockSubscriber(expected_endpoint_1, None, publisher_config.topic)
try:
time.sleep(0.1) # Let publishers start up
# Publish events from different ranks
batch_0 = create_test_events(2)
batch_1 = create_test_events(3)
pub_0.publish(batch_0)
pub_1.publish(batch_1)
# Receive events from rank 0
result_0 = sub_0.receive_one(timeout=200)
assert result_0 is not None, "No message received from rank 0"
seq_0, received_0 = result_0
# Receive events from rank 1
result_1 = sub_1.receive_one(timeout=200)
assert result_1 is not None, "No message received from rank 1"
seq_1, received_1 = result_1
# Verify DP rank tagging
assert received_0.data_parallel_rank == 0, (
f"Expected DP rank 0, got {received_0.data_parallel_rank}"
)
assert received_1.data_parallel_rank == 1, (
f"Expected DP rank 1, got {received_1.data_parallel_rank}"
)
# Verify event content is correct
assert len(received_0.events) == 2, "Wrong number of events from rank 0"
assert len(received_1.events) == 3, "Wrong number of events from rank 1"
finally:
pub_0.shutdown()
pub_1.shutdown()
sub_0.close()
sub_1.close()
def test_event_publisher_factory(random_port):
"""Test event publisher factory creation behavior under different configurations."""
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed.kv_events import ZmqEventPublisher
# test config is None
publisher = EventPublisherFactory.create(None, DP_RANK)
assert isinstance(publisher, NullEventPublisher)
publisher.shutdown()
# test disable kv cache events
config = KVEventsConfig(
enable_kv_cache_events=False,
publisher="zmq", # Even if zmq is specified, should return NullEventPublisher
endpoint="tcp://localhost:5557",
)
publisher = EventPublisherFactory.create(config, DP_RANK)
assert isinstance(publisher, NullEventPublisher)
publisher.shutdown()
# test zmq publisher
config = KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
endpoint=f"tcp://*:{random_port}",
replay_endpoint=f"tcp://*:{random_port + 100}",
)
publisher = EventPublisherFactory.create(config, DP_RANK + 1)
assert isinstance(publisher, ZmqEventPublisher)
resolved_config = publisher.get_publisher_config()
assert resolved_config.endpoint == f"tcp://*:{random_port + 1}"
assert resolved_config.replay_endpoint == f"tcp://*:{random_port + 101}"
assert config.endpoint == f"tcp://*:{random_port}"
publisher.shutdown()
# test unknown publisher
with pytest.raises(ValueError, match="Input should be"):
KVEventsConfig(
enable_kv_cache_events=True,
publisher="unknown_publisher",
endpoint="tcp://localhost:5557",
)
# test publisher not specified
config = KVEventsConfig(
enable_kv_cache_events=True,
# publisher not specified, should default to "zmq"
endpoint="tcp://localhost:5557",
)
publisher = EventPublisherFactory.create(config, DP_RANK)
assert isinstance(publisher, ZmqEventPublisher)
publisher.shutdown()
@pytest.mark.parametrize(
("endpoint", "expected"),
[("tcp://*:5557", "tcp://*:5558"), ("tcp://*:0", "tcp://*:0")],
)
def test_offset_endpoint_port_keeps_port_zero(endpoint, expected):
"""Port 0 is not offset: every rank asks the OS for its own port."""
from vllm.distributed.kv_events import ZmqEventPublisher
assert ZmqEventPublisher.offset_endpoint_port(endpoint, 1) == expected
@pytest.mark.parametrize("node_ip", ["127.0.0.1", "::1"])
def test_ephemeral_publishers_report_bound_endpoints(monkeypatch, node_ip):
"""Two ranks binding tcp://*:0 get distinct OS-assigned ports on this
node's IP and report endpoints a subscriber can use for events and replay."""
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed import kv_events
from vllm.utils.network_utils import split_zmq_path
from .conftest import MockSubscriber
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
config = KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
endpoint="tcp://*:0",
replay_endpoint="tcp://*:0",
topic="kv",
)
pubs = [EventPublisherFactory.create(config, rank) for rank in range(2)]
try:
resolved = [pub.get_publisher_config() for pub in pubs]
endpoints = [e for c in resolved for e in (c.endpoint, c.replay_endpoint)]
hosts_ports = {split_zmq_path(e)[1:] for e in endpoints}
assert {h for h, _ in hosts_ports} == {node_ip}
assert len(hosts_ports) == 4 and "0" not in {p for _, p in hosts_ports}
assert config.endpoint == "tcp://*:0"
sub = MockSubscriber(endpoints[::2], endpoints[1::2], topic="kv")
try:
# PUB/SUB drops messages sent before the subscription lands, so
# keep publishing until both ranks are heard.
ranks: set[int] = set()
deadline = time.time() + 10
while ranks != {0, 1} and time.time() < deadline:
for pub in pubs:
pub.publish(create_test_events(1))
while (result := sub.receive_one(timeout=100)) is not None:
ranks.add(result[1].data_parallel_rank)
assert ranks == {0, 1}
sub.request_replay(0, socket_idx=1)
assert sub.receive_replay(socket_idx=1)
finally:
sub.close()
finally:
for pub in pubs:
pub.shutdown()
@pytest.mark.parametrize("node_ip", ["0.0.0.0", "::"])
def test_ephemeral_bind_needs_a_node_ip(monkeypatch, node_ip):
"""tcp://*:0 is advertised as this node's IP; with no usable IP the
publisher refuses instead of reporting a wildcard."""
from vllm.config.kv_events import KVEventsConfig
from vllm.distributed import kv_events
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
config = KVEventsConfig(
enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://*:0"
)
with pytest.raises(ValueError, match="VLLM_HOST_IP"):
EventPublisherFactory.create(config, DP_RANK)
@@ -1,146 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm.distributed.device_communicators import cuda_communicator
from vllm.distributed.device_communicators import (
flashinfer_pcie_ipc_all_reduce as pcie_ipc,
)
from vllm.distributed.parallel_state import GroupCoordinator
def _uninitialized_comm() -> pcie_ipc.FlashInferPcieIpcAllReduce:
comm = pcie_ipc.FlashInferPcieIpcAllReduce.__new__(
pcie_ipc.FlashInferPcieIpcAllReduce
)
comm.disabled = False
comm.group = object()
comm.tune_group = object()
comm.device = torch.device("cuda:0")
comm.world_size = 4
comm.rank = 0
comm.workspace = None
comm.hidden_dim = 0
comm.dtype = None
return comm
def test_setup_uses_exact_graph_capacity(monkeypatch, tmp_path):
workspace = Mock()
factory = Mock(return_value=workspace)
monkeypatch.setattr(
pcie_ipc,
"flashinfer_comm",
SimpleNamespace(PcieIpcAllReduceWorkspace=factory),
)
monkeypatch.setattr(torch.accelerator, "synchronize", Mock())
comm = _uninitialized_comm()
cache = tmp_path / "tp4.json"
comm.setup(
hidden_dim=4096,
dtype=torch.bfloat16,
capture_sizes=[32, 8, 16, 32],
tune_cache=cache,
)
factory.assert_called_once_with(
group=comm.group,
max_numel=32 * 4096,
dtype=torch.bfloat16,
tune_batches=(8, 16, 32),
tune_cache=str(cache),
)
workspace.tune.assert_called_once_with(
[4096], dtype=torch.bfloat16, tune_group=comm.tune_group
)
workspace.prepare.assert_called_once_with(
[(8, 4096), (16, 4096), (32, 4096)], dtype=torch.bfloat16
)
assert workspace.rebind_stream.call_count == 2
@pytest.mark.parametrize(
("shape", "dtype", "contiguous", "expected"),
[
((16, 4096), torch.bfloat16, True, True),
((16, 2048), torch.bfloat16, True, False),
((16, 4096), torch.float16, True, False),
((16, 4096), torch.bfloat16, False, False),
],
)
def test_should_use_only_prepared_hidden_shape(shape, dtype, contiguous, expected):
comm = _uninitialized_comm()
comm.hidden_dim = 4096
comm.dtype = torch.bfloat16
comm.workspace = Mock()
comm.workspace.supports.return_value = True
inp = SimpleNamespace(
is_cuda=True,
is_contiguous=lambda: contiguous,
dim=lambda: len(shape),
shape=shape,
dtype=dtype,
)
assert comm.should_use(inp) is expected
def test_capture_rebinds_workspace_even_on_error(monkeypatch):
comm = _uninitialized_comm()
comm.workspace = Mock()
synchronize = Mock()
monkeypatch.setattr(torch.accelerator, "synchronize", synchronize)
with pytest.raises(RuntimeError, match="capture failed"), comm.capture():
raise RuntimeError("capture failed")
assert synchronize.call_count == 2
assert comm.workspace.rebind_stream.call_count == 2
def test_cuda_dispatch_prefers_pcie_ipc_before_existing_flashinfer(monkeypatch):
communicator = cuda_communicator.CudaCommunicator.__new__(
cuda_communicator.CudaCommunicator
)
communicator.pynccl_comm = Mock(world_size=4)
communicator.qr_comm = None
communicator.fi_pcie_ipc_ar_comm = Mock()
communicator.fi_pcie_ipc_ar_comm.should_use.return_value = True
expected = object()
communicator.fi_pcie_ipc_ar_comm.all_reduce.return_value = expected
communicator.fi_ar_comm = Mock()
communicator.aiter_ar_comm = None
communicator.ca_comm = None
communicator.symm_mem_comm = None
monkeypatch.setattr(
cuda_communicator, "should_nccl_symm_mem_allreduce", lambda *_: False
)
assert communicator.all_reduce(object()) is expected
communicator.fi_ar_comm.should_use_fi_ar.assert_not_called()
def test_group_destroy_releases_communicator_before_process_groups(monkeypatch):
events = []
coordinator = GroupCoordinator.__new__(GroupCoordinator)
coordinator.device_communicator = SimpleNamespace(
destroy=lambda: events.append("communicator")
)
coordinator.device_group = object()
coordinator.cpu_group = object()
coordinator.mq_broadcaster = None
monkeypatch.setattr(
torch.distributed,
"destroy_process_group",
lambda _group: events.append("process_group"),
)
coordinator.destroy()
assert events == ["communicator", "process_group", "process_group"]
@@ -1,238 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from pathlib import Path
import pytest
from tests.conftest import VllmRunner
from tests.utils import create_new_process_for_each_test
from vllm import SamplingParams, TokensPrompt
from vllm.config import CUDAGraphMode
from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
MODEL = "moonshotai/Kimi-Linear-48B-A3B-Instruct"
def _make_tiny_overrides() -> dict:
linear_attn_config = {
"full_attn_layers": [4],
"head_dim": 128,
"kda_layers": [1, 2, 3],
"num_heads": 8,
"short_conv_kernel_size": 4,
"use_full_rank_gate": True,
}
return {
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 0,
"vocab_size": 256,
"hidden_size": 256,
"head_dim": 32,
"intermediate_size": 512,
"num_hidden_layers": 4,
"num_attention_heads": 8,
"num_key_value_heads": 8,
"num_experts": None,
"num_experts_per_token": None,
"num_shared_experts": 0,
"q_lora_rank": 128,
"kv_lora_rank": 512,
"qk_nope_head_dim": 128,
"qk_rope_head_dim": 64,
"v_head_dim": 128,
"mla_use_nope": True,
"mla_use_output_gate": True,
"linear_attn_config": linear_attn_config,
"max_position_embeddings": 512,
"model_max_length": 512,
}
def _run_tiny_model(
vllm_runner: type[VllmRunner], dcp_size: int
) -> list[tuple[list[int], list[float]]]:
with vllm_runner(
model_name=MODEL,
skip_tokenizer_init=True,
load_format="dummy",
hf_overrides=_make_tiny_overrides(),
tensor_parallel_size=2,
decode_context_parallel_size=dcp_size,
cp_kv_cache_interleave_size=1,
distributed_executor_backend="mp",
dtype="bfloat16",
seed=0,
enforce_eager=True,
max_model_len=512,
max_num_seqs=8,
max_num_batched_tokens=1024,
gpu_memory_utilization=0.25,
enable_chunked_prefill=True,
enable_prefix_caching=False,
) as runner:
lengths = [1, 32, 33, 129]
prompts = [
TokensPrompt(
prompt_token_ids=[
3 + ((request_idx + token_idx) % 251) for token_idx in range(length)
]
)
for request_idx, length in enumerate(lengths)
]
outputs = runner.llm.generate(
prompts,
SamplingParams(
temperature=0,
max_tokens=8,
seed=0,
ignore_eos=True,
logprobs=20,
),
use_tqdm=False,
)
results = []
for request_output in outputs:
completion = request_output.outputs[0]
token_ids = list(completion.token_ids)
assert completion.logprobs is not None
selected_logprobs = [
step_logprobs[token_id].logprob
for token_id, step_logprobs in zip(token_ids, completion.logprobs)
]
results.append((token_ids, selected_logprobs))
return results
@create_new_process_for_each_test()
@pytest.mark.distributed(num_gpus=2)
def test_kimi_linear_dcp_tiny(
vllm_runner: type[VllmRunner],
num_gpus_available: int,
) -> None:
if num_gpus_available < 2:
pytest.skip("Need at least 2 GPUs")
baseline = _run_tiny_model(vllm_runner, dcp_size=1)
dcp = _run_tiny_model(vllm_runner, dcp_size=2)
assert [tokens for tokens, _ in dcp] == [tokens for tokens, _ in baseline]
logprob_drifts = []
for (_, baseline_logprobs), (_, dcp_logprobs) in zip(baseline, dcp):
for baseline_logprob, dcp_logprob in zip(baseline_logprobs, dcp_logprobs):
assert math.isfinite(baseline_logprob)
assert math.isfinite(dcp_logprob)
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
assert max(logprob_drifts) <= 1e-2
def _make_tiny_k3_config(model_dir: Path) -> str:
config = KimiK3Config(
text_config=_make_tiny_overrides(),
vision_config={
"vt_num_attention_heads": 2,
"vt_num_hidden_layers": 1,
"vt_hidden_size": 32,
"vt_intermediate_size": 64,
"qkv_hidden_size": 48,
},
architectures=["KimiK3ForConditionalGeneration"],
)
config.save_pretrained(model_dir)
return str(model_dir)
def _run_k3_partial_prefix_reuse(
vllm_runner: type[VllmRunner], model_name: str, dcp_size: int
) -> tuple[list[int], list[float], int]:
block_size = 256
common_prefix = [3 + (token_idx % 251) for token_idx in range(block_size)]
prime_prompt = TokensPrompt(prompt_token_ids=common_prefix + [17])
replay_prompt = TokensPrompt(prompt_token_ids=common_prefix + [18, 19])
sampling_params = SamplingParams(
temperature=0,
max_tokens=8,
seed=0,
ignore_eos=True,
logprobs=20,
)
with vllm_runner(
model_name=model_name,
skip_tokenizer_init=True,
load_format="dummy",
language_model_only=True,
tensor_parallel_size=2,
decode_context_parallel_size=dcp_size,
cp_kv_cache_interleave_size=1,
distributed_executor_backend="mp",
dtype="bfloat16",
seed=0,
max_model_len=512,
max_num_seqs=4,
max_num_batched_tokens=256,
gpu_memory_utilization=0.85,
block_size=block_size,
enable_chunked_prefill=True,
enable_prefix_caching=True,
) as runner:
assert (
runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
== CUDAGraphMode.FULL_AND_PIECEWISE
)
runner.llm.generate(
[prime_prompt],
sampling_params,
use_tqdm=False,
)
replay_output = runner.llm.generate(
[replay_prompt],
sampling_params,
use_tqdm=False,
)[0]
completion = replay_output.outputs[0]
token_ids = list(completion.token_ids)
assert len(token_ids) == sampling_params.max_tokens
assert completion.logprobs is not None
assert len(completion.logprobs) == sampling_params.max_tokens
selected_logprobs = [
step_logprobs[token_id].logprob
for token_id, step_logprobs in zip(token_ids, completion.logprobs)
]
return token_ids, selected_logprobs, replay_output.num_cached_tokens
@create_new_process_for_each_test()
@pytest.mark.distributed(num_gpus=2)
def test_kimi_k3_dcp_partial_prefix_reuse(
vllm_runner: type[VllmRunner],
num_gpus_available: int,
tmp_path: Path,
) -> None:
if num_gpus_available < 2:
pytest.skip("Need at least 2 GPUs")
model_name = _make_tiny_k3_config(tmp_path / "tiny-kimi-k3")
baseline_tokens, baseline_logprobs, baseline_cached = _run_k3_partial_prefix_reuse(
vllm_runner, model_name, dcp_size=1
)
dcp_tokens, dcp_logprobs, dcp_cached = _run_k3_partial_prefix_reuse(
vllm_runner, model_name, dcp_size=2
)
assert baseline_cached == 256
assert dcp_cached == 256
assert dcp_tokens == baseline_tokens
assert len(dcp_logprobs) == len(baseline_logprobs) == 8
logprob_drifts = []
for baseline_logprob, dcp_logprob in zip(baseline_logprobs, dcp_logprobs):
assert math.isfinite(baseline_logprob)
assert math.isfinite(dcp_logprob)
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
assert max(logprob_drifts) <= 1e-2
File diff suppressed because it is too large Load Diff
@@ -1,498 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Integration tests for MultiprocExecutor at the executor level.
This test directly tests the executor without going through the LLM interface,
focusing on executor initialization, RPC calls, and distributed execution.
"""
import multiprocessing
import os
import socket
import pytest
from tests.utils import multi_gpu_test
from vllm.config import VllmConfig
from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.executor import multiproc_executor
from vllm.v1.executor.multiproc_executor import MultiprocExecutor
MODEL = "facebook/opt-125m"
@pytest.mark.parametrize(
("local_world_size", "data_parallel_size_local", "expected_num_local_procs"),
[(1, 4, 4), (4, 1, 4), (2, 0, 2)],
)
def test_multiproc_executor_counts_all_local_dp_workers(
monkeypatch: pytest.MonkeyPatch,
local_world_size: int,
data_parallel_size_local: int,
expected_num_local_procs: int,
):
"""All colocated DP workers share the node's startup CPU budget."""
executor = object.__new__(MultiprocExecutor)
executor.world_size = local_world_size
executor.local_world_size = local_world_size
executor.parallel_config = type(
"ParallelConfig",
(),
{"data_parallel_size_local": data_parallel_size_local},
)()
monkeypatch.setattr(
executor,
"_get_parallel_sizes",
lambda: (local_world_size, 1, 1),
)
class StopExecutorInit(Exception):
pass
def capture_num_local_procs(num_local_procs: int):
assert num_local_procs == expected_num_local_procs
raise StopExecutorInit
monkeypatch.setattr(
multiproc_executor,
"set_multiprocessing_worker_envs",
capture_num_local_procs,
)
with pytest.raises(StopExecutorInit):
executor._init_executor()
executor._finalizer.detach()
def create_vllm_config(
tensor_parallel_size: int = 1,
pipeline_parallel_size: int = 1,
max_model_len: int = 256,
gpu_memory_utilization: float = 0.3,
distributed_executor_backend: str = "mp",
nnodes: int = 1,
node_rank: int = 0,
master_port: int = 0,
) -> VllmConfig:
"""Create a VllmConfig for testing using EngineArgs."""
engine_args = EngineArgs(
model=MODEL,
tensor_parallel_size=tensor_parallel_size,
pipeline_parallel_size=pipeline_parallel_size,
max_model_len=max_model_len,
gpu_memory_utilization=gpu_memory_utilization,
distributed_executor_backend=distributed_executor_backend,
enforce_eager=True,
)
vllm_config = engine_args.create_engine_config()
# Override distributed node settings if needed
if nnodes > 1 or node_rank > 0:
vllm_config.parallel_config.nnodes = nnodes
vllm_config.parallel_config.node_rank = node_rank
vllm_config.parallel_config.master_port = master_port
if nnodes > 1:
vllm_config.parallel_config.disable_custom_all_reduce = True
return vllm_config
def create_test_scheduler_output(num_requests: int = 1) -> SchedulerOutput:
"""Create a minimal SchedulerOutput for testing."""
# This is a simplified version - in practice you'd need proper
# SchedulerOutput construction based on the actual vLLM v1 API
return SchedulerOutput(
scheduled_new_reqs=[],
scheduled_resumed_reqs=[],
scheduled_running_reqs=[],
num_scheduled_tokens={},
total_num_scheduled_tokens=0,
)
def test_multiproc_executor_initialization():
"""Test that MultiprocExecutor can be initialized with proper config."""
vllm_config = create_vllm_config(
tensor_parallel_size=1,
pipeline_parallel_size=1,
)
# Create executor - this should initialize workers
executor = MultiprocExecutor(vllm_config=vllm_config)
# Verify executor properties
assert executor.world_size == 1, "World size should be 1 for single GPU"
assert executor.local_world_size == 1, "Local world size should be 1"
assert hasattr(executor, "workers"), "Executor should have workers"
assert len(executor.workers) == 1, "Should have 1 worker for single GPU"
# Clean up
executor.shutdown()
@multi_gpu_test(num_gpus=2)
def test_multiproc_executor_initialization_tensor_parallel():
"""Test MultiprocExecutor initialization with tensor parallelism."""
vllm_config = create_vllm_config(
tensor_parallel_size=2,
pipeline_parallel_size=1,
)
# Create executor
executor = MultiprocExecutor(vllm_config=vllm_config)
# Verify executor properties
assert executor.world_size == 2, "World size should be 2 for TP=2"
assert executor.local_world_size == 2, "Local world size should be 2"
assert len(executor.workers) == 2, "Should have 2 workers for TP=2"
# Verify output rank calculation
output_rank = executor._get_output_rank()
assert output_rank == 0, "Output rank should be 0 for TP=2, PP=1"
# Clean up
executor.shutdown()
@multi_gpu_test(num_gpus=2)
def test_multiproc_executor_collective_rpc():
"""Test collective RPC calls to all workers."""
vllm_config = create_vllm_config(
tensor_parallel_size=2,
pipeline_parallel_size=1,
)
# Create executor
executor = MultiprocExecutor(vllm_config=vllm_config)
try:
# Test check_health RPC - should work without errors
executor.check_health()
# Test that RPC works correctly
# Note: We're just testing that the RPC mechanism works,
# not testing actual model execution here
assert not executor.is_failed, "Executor should not be in failed state"
finally:
# Clean up
executor.shutdown()
def test_multiproc_executor_failure_callback():
"""Test failure callback registration and invocation."""
vllm_config = create_vllm_config(
tensor_parallel_size=1,
pipeline_parallel_size=1,
)
executor = MultiprocExecutor(vllm_config=vllm_config)
try:
# Test callback registration
callback_invoked = []
def test_callback():
callback_invoked.append(True)
# Register callback
executor.register_failure_callback(test_callback)
# Callback should not be invoked yet
assert len(callback_invoked) == 0, "Callback should not be invoked immediately"
# Simulate failure
executor.is_failed = True
# Register another callback - should be invoked immediately
executor.register_failure_callback(test_callback)
assert len(callback_invoked) == 1, (
"Callback should be invoked when executor is failed"
)
finally:
# Clean up
executor.shutdown()
@multi_gpu_test(num_gpus=2)
def test_multiproc_executor_worker_monitor():
"""Test that worker monitor is set up correctly."""
vllm_config = create_vllm_config(
tensor_parallel_size=2,
pipeline_parallel_size=1,
)
executor = MultiprocExecutor(vllm_config=vllm_config)
try:
# Verify all worker processes are alive
for worker in executor.workers:
assert worker.proc.is_alive(), f"Worker rank {worker.rank} should be alive"
# Verify executor is not in failed state
assert not executor.is_failed, "Executor should not be in failed state"
finally:
# Clean up
executor.shutdown()
# After shutdown, workers should be terminated
import time
time.sleep(0.5) # Give processes time to terminate
for worker in executor.workers:
assert not worker.proc.is_alive(), (
f"Worker rank {worker.rank} should terminate after shutdown"
)
@multi_gpu_test(num_gpus=2)
def test_multiproc_executor_get_response_message_queues():
"""Test message queue retrieval for different ranks."""
vllm_config = create_vllm_config(
tensor_parallel_size=2,
pipeline_parallel_size=1,
)
executor = MultiprocExecutor(vllm_config=vllm_config)
try:
# Get all message queues
all_queues = executor.get_response_mqs()
assert len(all_queues) == 2, "Should have 2 message queues for 2 workers"
# Get message queue for specific rank
rank0_queue = executor.get_response_mqs(unique_reply_rank=0)
assert len(rank0_queue) == 1, "Should have 1 message queue for rank 0"
rank1_queue = executor.get_response_mqs(unique_reply_rank=1)
assert len(rank1_queue) == 1, "Should have 1 message queue for rank 1"
finally:
# Clean up
executor.shutdown()
def test_multiproc_executor_shutdown_cleanup():
"""Test that shutdown properly cleans up resources."""
vllm_config = create_vllm_config(
tensor_parallel_size=1,
pipeline_parallel_size=1,
)
executor = MultiprocExecutor(vllm_config=vllm_config)
# Verify executor is set up
assert hasattr(executor, "workers"), "Executor should have workers"
assert len(executor.workers) > 0, "Should have at least one worker"
# Shutdown
executor.shutdown()
# Verify cleanup
import time
time.sleep(0.5) # Give processes time to terminate
for worker in executor.workers:
assert not worker.proc.is_alive(), "Worker processes should be terminated"
# Verify shutdown flag is set
assert executor.shutting_down, "Shutdown flag should be set"
# Multiple shutdowns should be safe (idempotent)
executor.shutdown()
executor.shutdown()
@multi_gpu_test(num_gpus=4)
def test_multiproc_executor_pipeline_parallel():
"""Test MultiprocExecutor with pipeline parallelism."""
vllm_config = create_vllm_config(
tensor_parallel_size=2,
pipeline_parallel_size=2,
)
executor = MultiprocExecutor(vllm_config=vllm_config)
try:
# Verify executor properties
assert executor.world_size == 4, "World size should be 4 for TP=2, PP=2"
assert len(executor.workers) == 4, "Should have 4 workers"
# Verify output rank calculation
# For TP=2, PP=2: output should be from the last PP stage (ranks 2-3)
# Specifically rank 2 (first rank of last PP stage)
output_rank = executor._get_output_rank()
assert output_rank == 2, "Output rank should be 2 (first rank of last PP stage)"
# V2 model runner uses one extra batch to overlap async scheduling.
expected_concurrent_batches = 2 + int(
vllm_config.scheduler_config.async_scheduling
and vllm_config.use_v2_model_runner
)
assert vllm_config.max_concurrent_batches == expected_concurrent_batches, (
"Max concurrent batches should follow the configured PP/async "
"scheduling policy"
)
finally:
# Clean up
executor.shutdown()
def test_multiproc_executor_properties():
"""Test various executor properties and configurations."""
vllm_config = create_vllm_config(
tensor_parallel_size=1,
pipeline_parallel_size=1,
)
executor = MultiprocExecutor(vllm_config=vllm_config)
try:
# Test supports_pp property
assert MultiprocExecutor.supports_pp is True, (
"MultiprocExecutor should support pipeline parallelism"
)
# Test world_size calculation
assert executor.world_size == (
executor.parallel_config.tensor_parallel_size
* executor.parallel_config.pipeline_parallel_size
), "World size should equal TP * PP"
# Test local_world_size calculation
assert executor.local_world_size == (
executor.parallel_config.world_size // executor.parallel_config.nnodes
), "Local world size should be world_size / nnodes"
finally:
# Clean up
executor.shutdown()
@multi_gpu_test(num_gpus=4)
def test_multiproc_executor_multi_node():
"""Test MultiprocExecutor with multi-node configuration.
This simulates 2 nodes with TP=4:
- Node 0 (rank 0): Uses GPUs 0,1 (device_control_env_var=0,1) with TP=2
- Node 1 (rank 1): Uses GPUs 2,3 (device_control_env_var=2,3) with TP=2
Total world_size = 4, nnodes = 2
"""
# Python 3.14+ changed default multiprocessing start method to 'forkserver'
# which cannot pickle nested functions. Use 'fork' for this test.
mp_ctx = multiprocessing.get_context("fork")
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("", 0))
port = s.getsockname()[1]
# symm_mem does not work for simulating multi instance in single node
os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0"
# Use the platform-specific device control env var (e.g.
# CUDA_VISIBLE_DEVICES on CUDA, ZE_AFFINITY_MASK on XPU) so that each
# node only sees its own devices on different platform.
device_control_env_var = current_platform.device_control_env_var
def run_node(node_rank: int, result_queue: multiprocessing.Queue, port: int):
"""Run a single node's executor."""
executor = None
try:
# Restrict visible devices for this node
if node_rank == 0:
os.environ[device_control_env_var] = "0,1"
else:
os.environ[device_control_env_var] = "2,3"
# Create config for this node
vllm_config = create_vllm_config(
tensor_parallel_size=4, # Total TP across all nodes
pipeline_parallel_size=1,
nnodes=2, # 2 nodes
node_rank=node_rank,
master_port=port, # same port
)
# Create executor for this node
executor = MultiprocExecutor(vllm_config=vllm_config)
# Verify node-specific properties
assert executor.world_size == 4, (
f"World size should be 4 on node {node_rank}"
)
assert executor.local_world_size == 2, (
f"Local world size should be 2 on node {node_rank}"
)
assert len(executor.workers) == 2, (
f"Should have 2 local workers on node {node_rank}"
)
# Verify worker ranks are correct for this node
expected_ranks = [node_rank * 2, node_rank * 2 + 1]
actual_ranks = sorted([w.rank for w in executor.workers])
assert actual_ranks == expected_ranks, (
f"Node {node_rank} should have workers "
f"with ranks {expected_ranks}, got {actual_ranks}"
)
# Verify all workers are alive
for worker in executor.workers:
assert worker.proc.is_alive(), (
f"Worker rank {worker.rank} should be alive on node {node_rank}"
)
# executor.gen
# Put success result in queue BEFORE shutdown to avoid hanging
result_queue.put({"node": node_rank, "success": True})
import time
time.sleep(2)
executor.shutdown()
except Exception as e:
# Put failure result in queue
result_queue.put({"node": node_rank, "success": False, "error": str(e)})
raise e
finally:
if executor is not None:
executor.shutdown()
# Create a queue to collect results from both processes
result_queue: multiprocessing.Queue[dict[str, int | bool]] = mp_ctx.Queue()
# Start both node processes
processes = []
for node_rank in range(2):
p = mp_ctx.Process(
target=run_node,
args=(node_rank, result_queue, port),
name=f"Node{node_rank}",
)
p.start()
processes.append(p)
# Wait for both processes to complete
all_completed = True
for p in processes:
p.join(timeout=60)
if p.is_alive():
p.terminate()
p.join(timeout=20)
if p.is_alive():
p.kill()
p.join()
all_completed = False
# Check results from both nodes
results: list[dict[str, int | bool]] = []
while len(results) < 2:
try:
result = result_queue.get(timeout=1)
results.append(result)
except Exception:
pass
assert all_completed, "Not all processes completed successfully"
assert len(results) == 2, f"Expected 2 results, got {len(results)}"
assert results[0]["success"], f"Node 0 failed: {results[0]}"
assert results[1]["success"], f"Node 1 failed: {results[1]}"
-367
View File
@@ -1,367 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
import typing
from types import SimpleNamespace
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
import vllm.envs as envs
from tests.utils import ensure_current_vllm_config
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.distributed.device_communicators.cuda_communicator import (
NCCL_DIRECT_SYMM_RS_OUTPUT_MIN_VERSION,
CudaCommunicator,
)
from vllm.distributed.device_communicators.pynccl import register_nccl_symmetric_ops
from vllm.distributed.device_communicators.pynccl_allocator import (
get_nccl_mem_pool,
is_symmetric_memory_enabled,
is_symmetric_memory_tensor,
)
from vllm.distributed.parallel_state import (
get_tp_group,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.platforms import current_platform
from vllm.utils.system_utils import update_environment_variables
torch.manual_seed(42)
random.seed(44)
test_size_elements = 4 * 1024 * 1024
def test_disabled_pynccl_does_not_allocate_symmetric_buffer():
communicator = object.__new__(CudaCommunicator)
communicator.pynccl_comm = SimpleNamespace(disabled=True)
assert (
communicator.get_symmetric_memory_buffer(
"test", (1,), torch.float32, torch.device("cuda")
)
is None
)
@pytest.mark.parametrize("use_aiter", [False, True])
def test_reduce_scatterv_preserves_caller_output(monkeypatch, use_aiter):
import vllm.distributed.device_communicators.cuda_communicator as cc
communicator = object.__new__(CudaCommunicator)
communicator.world_size = 2
communicator.rank_in_group = 0
input_ = torch.arange(12, dtype=torch.float32).view(4, 3)
output = torch.full((2, 3), float("nan"))
def reduce_scatter(out, inp):
assert out is output
out.copy_(inp.view(2, 2, 3).sum(dim=0))
communicator.pynccl_comm = SimpleNamespace(
disabled=False, nccl_version=22902, reduce_scatter=reduce_scatter
)
communicator._can_use_aiter_ag_rs = lambda sizes: use_aiter and sizes is None
communicator.aiter_ar_comm = SimpleNamespace(
should_custom_rs=lambda inp, dim: True,
custom_reduce_scatter=lambda inp, out, dim: reduce_scatter(out, inp),
)
monkeypatch.setattr(cc, "should_nccl_symm_mem_ag_rs", lambda: True)
monkeypatch.setattr(cc, "is_symmetric_memory_tensor", lambda tensor: False)
monkeypatch.setenv("VLLM_BATCH_INVARIANT", "0")
# Explicit uniform sizes must not stage an ordinary input into symmetric
# scratch, and AITER must write the supplied output after normalization.
result = communicator.reduce_scatterv_into_output(
input_, output, dim=0, sizes=[2, 2]
)
assert result.data_ptr() == output.data_ptr()
torch.testing.assert_close(output, input_.view(2, 2, 3).sum(dim=0))
def nccl_symm_mem_allreduce_worker(local_rank: int, world_size: int):
monkeypatch = pytest.MonkeyPatch()
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
dtype = torch.bfloat16
device = torch.device(f"cuda:{local_rank}")
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.set_default_dtype(dtype)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": "12345",
}
)
init_distributed_environment()
with ensure_current_vllm_config():
initialize_model_parallel(tensor_model_parallel_size=world_size)
cuda_communicator = typing.cast(
CudaCommunicator, get_tp_group().device_communicator
)
pynccl_comm = cuda_communicator.pynccl_comm
if get_nccl_mem_pool() is None:
pytest.skip(
"NCCL allocator compilation failed (probably missing NCCL headers)."
)
if not is_symmetric_memory_enabled():
pytest.skip("NCCL symmetric memory allreduce is disabled.")
register_nccl_symmetric_ops(pynccl_comm)
input = torch.randint(1, 23, (test_size_elements,), dtype=dtype, device=device)
input_clone = input.clone()
output = torch.ops.vllm.all_reduce_symmetric_with_copy(input)
assert output is not None
group = get_tp_group().device_group
dist.all_reduce(input_clone, group=group)
torch.testing.assert_close(output, input_clone, atol=2.5, rtol=0.1)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="NCCLSymmMemAllreduce is only available for CUDA platforms.",
)
@pytest.mark.parametrize("world_size", [2])
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
def test_nccl_symm_mem_allreduce(monkeypatch: pytest.MonkeyPatch, world_size):
if world_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
# Enable SymmMemCommunicator
monkeypatch.setenv("VLLM_USE_NCCL_SYMM_MEM", "1")
monkeypatch.setenv("NCCL_NVLS_ENABLE", "1")
monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1")
mp.spawn(nccl_symm_mem_allreduce_worker, args=(world_size,), nprocs=world_size)
cleanup_dist_env_and_memory()
def nccl_symm_mem_allgather_worker(local_rank: int, world_size: int):
monkeypatch = pytest.MonkeyPatch()
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
dtype = torch.bfloat16
device = torch.device(f"cuda:{local_rank}")
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.set_default_dtype(dtype)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": "12346",
}
)
init_distributed_environment()
with ensure_current_vllm_config():
initialize_model_parallel(tensor_model_parallel_size=world_size)
cuda_communicator = typing.cast(
CudaCommunicator, get_tp_group().device_communicator
)
if get_nccl_mem_pool() is None:
pytest.skip(
"NCCL allocator compilation failed (probably missing NCCL headers)."
)
if not is_symmetric_memory_enabled():
pytest.skip("NCCL symmetric memory is disabled.")
per_rank_size = test_size_elements // world_size
input_tensor = torch.randint(
1, 23, (per_rank_size,), dtype=dtype, device=device
)
output = cuda_communicator.all_gatherv(input_tensor, dim=0)
group = get_tp_group().device_group
expected = torch.empty(test_size_elements, dtype=dtype, device=device)
dist.all_gather_into_tensor(expected, input_tensor, group=group)
torch.testing.assert_close(output, expected, atol=0.0, rtol=0.0)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="NCCL symmetric memory is only available for CUDA platforms.",
)
@pytest.mark.parametrize("world_size", [2])
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
def test_nccl_symm_mem_allgather(monkeypatch: pytest.MonkeyPatch, world_size):
if world_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
monkeypatch.setenv("VLLM_USE_NCCL_SYMM_MEM", "1")
monkeypatch.setenv("NCCL_NVLS_ENABLE", "1")
monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1")
mp.spawn(nccl_symm_mem_allgather_worker, args=(world_size,), nprocs=world_size)
cleanup_dist_env_and_memory()
def nccl_symm_mem_reduce_scatter_worker(local_rank: int, world_size: int):
monkeypatch = pytest.MonkeyPatch()
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
dtype = torch.bfloat16
device = torch.device(f"cuda:{local_rank}")
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.set_default_dtype(dtype)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": "12347",
}
)
init_distributed_environment()
with ensure_current_vllm_config():
initialize_model_parallel(tensor_model_parallel_size=world_size)
cuda_communicator = typing.cast(
CudaCommunicator, get_tp_group().device_communicator
)
if get_nccl_mem_pool() is None:
pytest.skip(
"NCCL allocator compilation failed (probably missing NCCL headers)."
)
if not is_symmetric_memory_enabled():
pytest.skip("NCCL symmetric memory is disabled.")
per_rank_size = test_size_elements // world_size
ordinary_input = torch.randint(
1, 23, (test_size_elements,), dtype=dtype, device=device
)
ordinary_clone = ordinary_input.clone()
ordinary_output = cuda_communicator.reduce_scatter(ordinary_input, dim=0)
group = get_tp_group().device_group
ordinary_expected = torch.empty(per_rank_size, dtype=dtype, device=device)
dist.reduce_scatter_tensor(ordinary_expected, ordinary_clone, group=group)
torch.testing.assert_close(
ordinary_output, ordinary_expected, atol=2.5, rtol=0.1
)
assert is_symmetric_memory_tensor(ordinary_output)
# Both calls reuse rs_out; validate the first result before overwriting it.
for sizes in (None, [per_rank_size] * world_size):
ordinary_v_output = cuda_communicator.reduce_scatterv(
ordinary_input, dim=0, sizes=sizes
)
torch.testing.assert_close(
ordinary_v_output, ordinary_expected, atol=2.5, rtol=0.1
)
assert is_symmetric_memory_tensor(ordinary_v_output)
pynccl_comm = cuda_communicator.pynccl_comm
assert pynccl_comm is not None
if pynccl_comm.nccl_version < NCCL_DIRECT_SYMM_RS_OUTPUT_MIN_VERSION:
return
from vllm.v1.worker import ubatching
m.setattr(ubatching, "dbo_current_ubatch_id", lambda: 0)
ubatch0 = cuda_communicator._get_symm_scratch("ag_out", (128,), dtype, device)
ubatch0.fill_(1)
m.setattr(ubatching, "dbo_current_ubatch_id", lambda: 1)
ubatch1 = cuda_communicator._get_symm_scratch("ag_out", (128,), dtype, device)
ubatch1.fill_(2)
assert ubatch0.data_ptr() != ubatch1.data_ptr()
assert (ubatch0 == 1).all()
m.setattr(ubatching, "dbo_current_ubatch_id", lambda: 0)
smaller = cuda_communicator._get_symm_scratch("ag_out", (64,), dtype, device)
assert smaller.shape == (64,)
assert smaller.data_ptr() == ubatch0.data_ptr()
grown = cuda_communicator._get_symm_scratch("ag_out", (129,), dtype, device)
assert grown.shape == (129,)
assert grown.data_ptr() != ubatch0.data_ptr()
reused = cuda_communicator._get_symm_scratch("ag_out", (200,), dtype, device)
assert reused.data_ptr() == grown.data_ptr()
input_tensor = cuda_communicator._get_symm_scratch(
"test_graph_rs_input", (test_size_elements,), dtype, device
)
input_tensor.random_(1, 23)
input_clone = input_tensor.clone()
output = torch.empty(per_rank_size, dtype=dtype, device=device)
assert is_symmetric_memory_tensor(input_tensor)
assert not is_symmetric_memory_tensor(output)
result = cuda_communicator.reduce_scatterv_into_output(
input_tensor,
output,
dim=0,
sizes=[per_rank_size] * world_size,
)
assert result.data_ptr() == output.data_ptr()
expected = torch.empty_like(output)
dist.reduce_scatter_tensor(expected, input_clone, group=group)
torch.testing.assert_close(result, expected, atol=2.5, rtol=0.1)
dist.barrier(group=get_tp_group().cpu_group)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_result = cuda_communicator.reduce_scatterv_into_output(
input_tensor,
output,
dim=0,
sizes=[per_rank_size] * world_size,
)
assert graph_result.data_ptr() == output.data_ptr()
input_tensor.random_(24, 47)
input_clone.copy_(input_tensor)
dist.reduce_scatter_tensor(expected, input_clone, group=group)
captured_input_ptr = input_tensor.data_ptr()
del input_tensor
grown_input = cuda_communicator._get_symm_scratch(
"test_graph_rs_input", (test_size_elements + 1,), dtype, device
)
assert grown_input.data_ptr() != captured_input_ptr
retired = cuda_communicator.__dict__["_retired_symm_scratch_bufs"]
assert any(
buf.data_ptr() == captured_input_ptr
for buffers in retired.values()
for buf in buffers
)
output.zero_()
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(output, expected, atol=2.5, rtol=0.1)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="NCCL symmetric memory is only available for CUDA platforms.",
)
@pytest.mark.parametrize("world_size", [2])
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
def test_nccl_symm_mem_reduce_scatter(monkeypatch: pytest.MonkeyPatch, world_size):
if world_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
monkeypatch.setenv("VLLM_USE_NCCL_SYMM_MEM", "1")
monkeypatch.setenv("NCCL_NVLS_ENABLE", "1")
monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1")
mp.spawn(nccl_symm_mem_reduce_scatter_worker, args=(world_size,), nprocs=world_size)
cleanup_dist_env_and_memory()
-41
View File
@@ -1,41 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.platforms import current_platform
from ..utils import compare_two_settings, create_new_process_for_each_test
@pytest.mark.parametrize(
"PP_SIZE, MODEL_NAME",
[
(2, "JackFram/llama-160m"),
],
)
@pytest.mark.parametrize(
"ATTN_BACKEND",
[None] if current_platform.is_rocm() else ["FLASH_ATTN"],
)
@create_new_process_for_each_test()
def test_pp_cudagraph(
PP_SIZE: int,
MODEL_NAME: str,
ATTN_BACKEND: str | None,
):
cudagraph_args = [
# use half precision for speed and memory savings in CI environment
"--dtype",
"float16",
"--pipeline-parallel-size",
str(PP_SIZE),
"--distributed-executor-backend",
"mp",
]
# On ROCm, defer to the platform attention selector instead of forcing a backend.
if ATTN_BACKEND is not None:
cudagraph_args.append(f"--attention-backend={ATTN_BACKEND}")
eager_args = cudagraph_args + ["--enforce-eager"]
compare_two_settings(MODEL_NAME, eager_args, cudagraph_args)
-664
View File
@@ -1,664 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import multiprocessing
import random
import sys
import types
from types import SimpleNamespace
import pytest
import ray
import torch
import torch.distributed as dist
import vllm._aiter_ops as aiter_ops
import vllm.distributed as vllm_distributed
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
from vllm.distributed.device_communicators.quick_all_reduce import (
KB,
MB,
QuickAllReduce,
QuickReduceRegime,
)
from vllm.distributed.parallel_state import get_tp_group, graph_capture
from vllm.envs import disable_envs_cache
from vllm.platforms import current_platform
from ..utils import (
ensure_model_parallel_initialized,
init_test_distributed_environment,
multi_process_parallel,
set_random_seed,
)
def on_gfx942() -> bool:
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx942 as rocm_on_gfx942
return rocm_on_gfx942()
return False
def _quickreduce_tolerance(quant_level: QuickReduceRegime) -> dict:
# With INT3 inputs of absmax 23, per-rank error peaks at 2.75,
# doubled by the TP2 sum = 5.5.
if quant_level == QuickReduceRegime.INT3:
return {"atol": 5.5, "rtol": 0.1}
return {"atol": 2.5, "rtol": 0.1}
set_random_seed(42)
_test_size_rng = random.Random(44)
# Size over 8MB is sufficient for custom quick allreduce.
test_sizes = [
_test_size_rng.randint(8 * 1024 * 1024, 10 * 1024 * 1024) for _ in range(8)
]
for i, v in enumerate(test_sizes):
test_sizes[i] -= v % 8
def _assert_quickreduce(fa, inp):
assert fa is not None
assert not fa.disabled
assert fa.should_quick_allreduce(inp)
@pytest.fixture
def envs_cache_disabled():
disable_envs_cache()
yield
disable_envs_cache()
def _make_quick_allreduce_for_test(
min_size_mb: int | None = None,
quantization_min_size: int | None = None,
) -> QuickAllReduce:
quick_reduce = QuickAllReduce.__new__(QuickAllReduce)
quick_reduce.disabled = False
quick_reduce.qr_max_size = 16 * MB
quick_reduce.qr_min_size = min_size_mb * MB if min_size_mb is not None else None
quick_reduce.qr_quant_level = QuickReduceRegime.INT4
quick_reduce.qr_quantization_min_size = quantization_min_size
quick_reduce.use_fp16_kernels = False
quick_reduce.world_size = 2
return quick_reduce
def test_should_quick_allreduce_uses_builtin_min_size_when_unset():
quick_reduce = _make_quick_allreduce_for_test(min_size_mb=None)
below_builtin_min = torch.empty(MB // 4, dtype=torch.float16)
at_builtin_min = torch.empty(MB // 2, dtype=torch.float16)
assert not quick_reduce.should_quick_allreduce(below_builtin_min)
assert quick_reduce.should_quick_allreduce(at_builtin_min)
def test_should_quick_allreduce_uses_min_size_override():
quick_reduce = _make_quick_allreduce_for_test(min_size_mb=0)
below_builtin_min = torch.empty(8, dtype=torch.float16)
assert quick_reduce.should_quick_allreduce(below_builtin_min)
def test_quick_allreduce_min_size_env_unset(
monkeypatch: pytest.MonkeyPatch,
envs_cache_disabled,
):
monkeypatch.delenv("VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB", raising=False)
assert QuickAllReduce._get_qr_min_size(qr_max_size=16 * MB) is None
def test_quick_allreduce_min_size_env_converts_mb_to_bytes(
monkeypatch: pytest.MonkeyPatch,
envs_cache_disabled,
):
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB", "4")
assert QuickAllReduce._get_qr_min_size(qr_max_size=16 * MB) == 4 * MB
def test_quick_allreduce_min_size_env_rejects_negative(
monkeypatch: pytest.MonkeyPatch,
envs_cache_disabled,
):
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB", "-1")
with pytest.raises(ValueError, match="must be non-negative"):
QuickAllReduce._get_qr_min_size(qr_max_size=16 * MB)
def test_quick_allreduce_min_size_env_allows_equal_to_max(
monkeypatch: pytest.MonkeyPatch,
envs_cache_disabled,
):
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB", "16")
assert QuickAllReduce._get_qr_min_size(qr_max_size=16 * MB) == 16 * MB
def test_quick_allreduce_min_size_env_rejects_larger_than_max(
monkeypatch: pytest.MonkeyPatch,
envs_cache_disabled,
):
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB", "17")
with pytest.raises(ValueError, match="effective QuickReduce max size"):
QuickAllReduce._get_qr_min_size(qr_max_size=16 * MB)
def test_quick_allreduce_quantization_min_size_env_unset(
monkeypatch: pytest.MonkeyPatch,
envs_cache_disabled,
):
monkeypatch.delenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB", raising=False)
assert QuickAllReduce._get_qr_quantization_min_size() is None
def test_quick_allreduce_quantization_min_size_env_converts_kb_to_bytes(
monkeypatch: pytest.MonkeyPatch,
envs_cache_disabled,
):
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB", "2048")
assert QuickAllReduce._get_qr_quantization_min_size() == 2048 * KB
def test_quick_allreduce_quantization_min_size_env_rejects_negative(
monkeypatch: pytest.MonkeyPatch,
envs_cache_disabled,
):
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB", "-1")
with pytest.raises(ValueError, match="must be non-negative"):
QuickAllReduce._get_qr_quantization_min_size()
def test_quick_allreduce_quantization_min_size_unset_uses_configured_codec():
quick_reduce = _make_quick_allreduce_for_test(quantization_min_size=None)
inp = torch.empty(8, dtype=torch.float16)
assert quick_reduce._get_qr_quant_level(inp) == QuickReduceRegime.INT4.value
def test_quick_allreduce_quantization_min_size_uses_fp_below_threshold():
quick_reduce = _make_quick_allreduce_for_test(quantization_min_size=2048)
inp = torch.empty(1024 // 2, dtype=torch.float16)
assert quick_reduce._get_qr_quant_level(inp) == QuickReduceRegime.FP.value
def test_quick_allreduce_quantization_min_size_uses_configured_codec_at_threshold():
quick_reduce = _make_quick_allreduce_for_test(quantization_min_size=2048)
inp = torch.empty(2048 // 2, dtype=torch.float16)
assert quick_reduce._get_qr_quant_level(inp) == QuickReduceRegime.INT4.value
def test_quick_allreduce_quantization_min_size_does_not_change_eligibility():
quick_reduce = _make_quick_allreduce_for_test(quantization_min_size=2 * MB)
below_builtin_min = torch.empty(MB // 4, dtype=torch.float16)
at_builtin_min = torch.empty(MB // 2, dtype=torch.float16)
assert not quick_reduce.should_quick_allreduce(below_builtin_min)
assert quick_reduce.should_quick_allreduce(at_builtin_min)
def test_quick_allreduce_passes_dynamic_quant_level(
monkeypatch: pytest.MonkeyPatch,
):
quick_reduce = _make_quick_allreduce_for_test(quantization_min_size=2 * KB)
quick_reduce._ptr = object()
inp = torch.empty(KB // 2, dtype=torch.float16)
called_quant_level = None
def fake_qr_all_reduce(
fa,
inp,
out,
quant_level,
cast_bf2half,
):
nonlocal called_quant_level
called_quant_level = quant_level
monkeypatch.setattr(ops, "qr_all_reduce", fake_qr_all_reduce)
quick_reduce.quick_all_reduce(inp)
assert called_quant_level == QuickReduceRegime.FP.value
def test_rocm_aiter_fused_rmsnorm_uses_aiter_qr_rmsnorm_for_prefill(
monkeypatch: pytest.MonkeyPatch,
):
class FakeAiterAllReduce:
aiter_ca = SimpleNamespace(world_size=2, fully_connected=True)
def use_1stage_fused_ar_rms(self, inp):
return False
def custom_fused_ar_rms(self, *args, **kwargs):
raise AssertionError("expected fused QR+RMSNorm dispatch")
class FakeQuickReduce:
disabled = False
def __init__(self):
self.checked_input = None
def should_quick_allreduce(self, inp):
self.checked_input = inp
return True
def _get_qr_quant_level(self, inp):
return QuickReduceRegime.INT4.value
qr_comm = FakeQuickReduce()
device_comm = SimpleNamespace(cpu_group=object(), qr_comm=qr_comm)
monkeypatch.setattr(
vllm_distributed,
"get_tp_group",
lambda: SimpleNamespace(device_communicator=device_comm),
)
monkeypatch.setattr(
aiter_ops.rocm_aiter_ops,
"get_aiter_allreduce",
lambda: FakeAiterAllReduce(),
)
monkeypatch.setattr(aiter_ops.dist, "get_world_size", lambda group: 2)
monkeypatch.setattr(aiter_ops.dist, "get_rank", lambda group: 0)
monkeypatch.setattr(
aiter_ops.dist,
"all_gather_object",
lambda handles, handle, group: handles.__setitem__(0, handle),
)
calls = []
fake_aiter = types.SimpleNamespace(
init_custom_qr=lambda rank, world_size, qr_max_size: 123,
qr_get_handle=lambda ptr: b"handle",
qr_open_handles=lambda ptr, handles: None,
)
def fake_qr_all_reduce_rmsnorm(
ptr,
inp,
residual,
residual_out,
out,
weight,
eps,
hidden_dim,
quant_level,
cast_bf2half,
):
calls.append(
{
"ptr": ptr,
"inp": inp,
"residual": residual,
"residual_out": residual_out,
"out": out,
"weight": weight,
"eps": eps,
"hidden_dim": hidden_dim,
"quant_level": quant_level,
"cast_bf2half": cast_bf2half,
}
)
out.copy_(inp)
residual_out.copy_(residual)
fake_aiter.qr_all_reduce_rmsnorm = fake_qr_all_reduce_rmsnorm
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setattr(envs, "VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", None)
monkeypatch.setattr(envs, "VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", True)
# token_num > 80 forces vLLM's AITER fused AR+RMS wrapper onto its
# non-1stage prefill path. hidden_dim=4096 bf16 gives an 8 KiB row, so four
# independent RMSNorm rows fit in one 32 KiB QR tile.
inp = torch.randn(81, 4096, dtype=torch.bfloat16)
residual = torch.randn_like(inp)
weight = torch.randn(4096, dtype=torch.bfloat16)
out, residual_out = aiter_ops._rocm_aiter_fused_allreduce_rmsnorm_impl(
inp, residual, weight, 1e-6
)
assert qr_comm.checked_input is inp
assert len(calls) == 1
assert calls[0]["ptr"] == 123
assert calls[0]["inp"] is inp
assert calls[0]["residual"] is residual
assert calls[0]["out"] is out
assert calls[0]["residual_out"] is residual_out
assert calls[0]["weight"] is weight
assert calls[0]["hidden_dim"] == 4096
assert calls[0]["quant_level"] == QuickReduceRegime.INT4.value
assert calls[0]["cast_bf2half"] is True
torch.testing.assert_close(out, inp)
torch.testing.assert_close(residual_out, residual)
def test_rocm_aiter_fused_rmsnorm_keeps_1stage_decode_path(
monkeypatch: pytest.MonkeyPatch,
):
calls = []
class FakeAiterCA:
world_size = 2
fully_connected = True
def custom_fused_ar_rms(
self, inp, residual, weight, epsilon, *, use_1stage, gemma_norm
):
calls.append(
{
"inp": inp,
"residual": residual,
"weight": weight,
"eps": epsilon,
"use_1stage": use_1stage,
"gemma_norm": gemma_norm,
}
)
return inp + 1, residual + 1
class FakeAiterAllReduce:
aiter_ca = FakeAiterCA()
def use_1stage_fused_ar_rms(self, inp):
return True
monkeypatch.setattr(
aiter_ops.rocm_aiter_ops,
"get_aiter_allreduce",
lambda: FakeAiterAllReduce(),
)
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
inp = torch.randn(4, 4096, dtype=torch.bfloat16)
residual = torch.randn_like(inp)
weight = torch.randn(4096, dtype=torch.bfloat16)
out, residual_out = aiter_ops._rocm_aiter_fused_allreduce_rmsnorm_impl(
inp, residual, weight, 1e-6
)
assert len(calls) == 1
assert calls[0]["inp"] is inp
assert calls[0]["residual"] is residual
assert calls[0]["weight"] is weight
assert calls[0]["use_1stage"] is True
assert calls[0]["gemma_norm"] is False
torch.testing.assert_close(out, inp + 1)
torch.testing.assert_close(residual_out, residual + 1)
@ray.remote(num_gpus=1, max_calls=1)
def graph_quickreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
m.delenv("ROCR_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
group = get_tp_group().device_group
fa = get_tp_group().device_communicator.qr_comm
# A small all_reduce for warmup.
# this is needed because device communicators might be created lazily
# (e.g. NCCL). This will ensure that the communicator is initialized
# before any communication happens, so that this group can be used for
# graph capture immediately.
data = torch.zeros(1)
data = data.to(device=device)
torch.distributed.all_reduce(data, group=group)
torch.accelerator.synchronize()
del data
# we use the first group to communicate once
# and the second group to communicate twice
# and so on
# this is used to demonstrate that each group can
# communicate independently
num_communication = rank // tp_size + 1
for sz in test_sizes:
for dtype in [torch.float16, torch.bfloat16]:
with graph_capture(device=device) as graph_capture_context:
device_idx = torch.accelerator.current_device_index()
inp1 = torch.randint(1, 23, (sz,), dtype=dtype, device=device_idx)
inp2 = torch.randint(-23, 1, (sz,), dtype=dtype, device=device_idx)
_assert_quickreduce(fa, inp1)
_assert_quickreduce(fa, inp2)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
for _ in range(num_communication):
out1 = tensor_model_parallel_all_reduce(inp1)
dist.all_reduce(inp1, group=group)
out2 = tensor_model_parallel_all_reduce(inp2)
dist.all_reduce(inp2, group=group)
graph.replay()
tol = _quickreduce_tolerance(fa.qr_quant_level)
torch.testing.assert_close(out1, inp1, **tol)
torch.testing.assert_close(out2, inp2, **tol)
@ray.remote(num_gpus=1, max_calls=1)
def eager_quickreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
m.delenv("ROCR_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
# Size over 8MB is sufficient for custom quick allreduce.
sz = 16 * 1024 * 1024
fa = get_tp_group().device_communicator.qr_comm
inp = torch.tensor(
[1.0 * ((i) % 23) for i in range(sz)], dtype=torch.float16, device=device
)
_assert_quickreduce(fa, inp)
out = fa.quick_all_reduce(inp)
tol = _quickreduce_tolerance(fa.qr_quant_level)
torch.testing.assert_close(out, inp * tp_size, **tol)
inp = torch.tensor(
[1.0 * ((i) % 23) for i in range(sz)], dtype=torch.bfloat16, device=device
)
_assert_quickreduce(fa, inp)
out = fa.quick_all_reduce(inp)
torch.testing.assert_close(out, inp * tp_size, **tol)
@ray.remote(num_gpus=1, max_calls=1)
def bf16_cast_quickreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
m.delenv("ROCR_VISIBLE_DEVICES", raising=False)
m.setenv("VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", "1")
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
sz = 16 * 1024 * 1024
fa = get_tp_group().device_communicator.qr_comm
inp = torch.tensor(
[1.0 * (i % 23) for i in range(sz)], dtype=torch.bfloat16, device=device
)
_assert_quickreduce(fa, inp)
assert fa.use_fp16_kernels
out = fa.quick_all_reduce(inp)
torch.testing.assert_close(out, inp * tp_size, atol=2.5, rtol=0.1)
@pytest.mark.skipif(
not current_platform.is_rocm(), reason="only test quick allreduce for rocm"
)
@pytest.mark.parametrize("quant_mode", ["FP", "INT8", "INT6", "INT4", "INT3"])
@pytest.mark.parametrize("tp_size", [2])
@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
@pytest.mark.parametrize("test_target", [graph_quickreduce, eager_quickreduce])
def test_custom_quick_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pipeline_parallel_size,
test_target,
quant_mode,
):
world_size = tp_size * pipeline_parallel_size
if world_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
if test_target is graph_quickreduce and on_gfx942():
pytest.xfail(
"CUDA graph capture with quick reduce hits "
"hipErrorStreamCaptureInvalidated on gfx942"
)
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", quant_mode)
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
@pytest.mark.skipif(
not current_platform.is_rocm(), reason="only test quick allreduce for rocm"
)
def test_custom_quick_allreduce_bf16_cast(monkeypatch: pytest.MonkeyPatch):
if torch.accelerator.device_count() < 2:
pytest.skip("Not enough GPUs to run the test.")
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", "FP")
multi_process_parallel(monkeypatch, 2, 1, bf16_cast_quickreduce)
def qr_variable_input(rank, world_size):
"""When the tensor parallelism is set to 4 or 8, frequent changes
in the input shape can cause QuickReduce to hang (this issue
has been observed with the gpt_oss model).
"""
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
qr_max_size = None # MB
_ptr = ops.init_custom_qr(rank, world_size, qr_max_size)
ranks = []
for i in range(world_size):
ranks.append(i)
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
dist.init_process_group(
backend="cpu:gloo,cuda:nccl",
init_method="tcp://127.0.0.1:29500",
rank=rank,
world_size=world_size,
device_id=device,
)
else:
dist.init_process_group(
backend="nccl",
init_method="tcp://127.0.0.1:29500",
rank=rank,
world_size=world_size,
)
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
cpu_group = torch.distributed.split_group(
split_ranks=[ranks], backend="cpu:gloo,cuda:nccl"
)
else:
cpu_group = torch.distributed.new_group(ranks, backend="nccl")
handle = ops.qr_get_handle(_ptr)
world_size = dist.get_world_size(group=cpu_group)
handles = [None] * world_size
dist.all_gather_object(handles, handle, group=cpu_group)
ops.qr_open_handles(_ptr, handles)
num = 1
s1 = 1024
while num < 50000: # 50000 is sufficient to identify issues.
dtype = torch.float16
device_idx = torch.accelerator.current_device_index()
if num % 2 == 0:
s2 = 1024
inp1 = torch.zeros((s1, s2), dtype=dtype, device=device_idx)
else:
s2 = 2048
inp1 = torch.ones((s1, s2), dtype=dtype, device=device_idx)
result = torch.empty_like(inp1)
# FP = 0 INT8 = 1 INT6 = 2 INT4 = 3 INT3 = 4
ops.qr_all_reduce(_ptr, inp1, result, 3, cast_bf2half=True)
try:
if inp1[0, 0] == 0:
assert torch.all(result == 0)
else:
assert torch.all(result == world_size)
except AssertionError:
print("Assertion failed! Allreduce results are incorrect.")
raise
num += 1
@pytest.mark.skipif(
not current_platform.is_rocm(), reason="only test quick allreduce for rocm"
)
@pytest.mark.parametrize("tp_size", [4, 8])
@pytest.mark.parametrize("pipeline_parallel_size", [1])
def test_custom_quick_allreduce_variable_input(tp_size, pipeline_parallel_size):
world_size = tp_size * pipeline_parallel_size
if world_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
multiprocessing.set_start_method("spawn", force=True)
# 60s is enough
timeout = 60
processes = []
for rank in range(tp_size):
p = multiprocessing.Process(target=qr_variable_input, args=(rank, tp_size))
p.start()
processes.append((rank, p))
for rank, p in processes:
p.join(timeout=timeout)
if p.is_alive():
for r, proc in processes:
if proc.is_alive():
proc.terminate()
proc.join()
raise RuntimeError(f"QuickReduce hang detected after {timeout} seconds!")
if __name__ == "__main__":
test_custom_quick_allreduce_variable_input(tp_size=4, pipeline_parallel_size=1)
-407
View File
@@ -1,407 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Integration tests for RayExecutorV2 at the executor level.
Validates executor initialization, placement group support, RPC calls,
and distributed execution with various TP/PP configurations.
"""
import errno
import gc
import socket
import threading
from types import SimpleNamespace
from unittest.mock import patch
from urllib.parse import urlsplit
import pytest
import ray
from torch.distributed import TCPStore
from vllm import LLM
from vllm.config import VllmConfig
from vllm.engine.arg_utils import EngineArgs
from vllm.v1.executor import ray_executor_v2
from vllm.v1.executor.ray_executor_v2 import RayExecutorV2
pytestmark = pytest.mark.usefixtures("enable_ray_v2_backend")
MODEL = "facebook/opt-125m"
def create_vllm_config(
tensor_parallel_size: int = 1,
pipeline_parallel_size: int = 1,
max_model_len: int = 256,
gpu_memory_utilization: float = 0.3,
placement_group=None,
) -> VllmConfig:
engine_args = EngineArgs(
model=MODEL,
tensor_parallel_size=tensor_parallel_size,
pipeline_parallel_size=pipeline_parallel_size,
max_model_len=max_model_len,
gpu_memory_utilization=gpu_memory_utilization,
distributed_executor_backend="ray",
enforce_eager=True,
)
vllm_config = engine_args.create_engine_config()
if placement_group is not None:
vllm_config.parallel_config.placement_group = placement_group
return vllm_config
def ensure_ray_initialized():
if not ray.is_initialized():
ray.init(ignore_reinit_error=True)
@pytest.fixture
def create_placement_group(request):
ensure_ray_initialized()
num_gpus = request.param
bundles = [{"GPU": 1, "CPU": 1} for _ in range(num_gpus)]
pg = ray.util.placement_group(bundles, strategy="PACK")
ray.get(pg.ready())
yield pg
ray.util.remove_placement_group(pg)
@pytest.fixture
def executor(request):
"""Create a RayExecutorV2 and shut it down after the test."""
executor = RayExecutorV2(vllm_config=request.param)
yield executor
executor.shutdown()
def assert_executor(executor, tp_size, pp_size):
"""Common assertions for executor initialization tests."""
world_size = tp_size * pp_size
expected_output_rank = (pp_size - 1) * tp_size
assert executor.world_size == world_size
assert len(executor.ray_worker_handles) == world_size
assert len(executor.response_mqs) == world_size
assert executor._get_output_rank() == expected_output_rank
if pp_size > 1:
expected_concurrent_batches = pp_size + int(
executor.vllm_config.scheduler_config.async_scheduling
and executor.vllm_config.use_v2_model_runner
)
assert (
executor.vllm_config.max_concurrent_batches == expected_concurrent_batches
)
executor.check_health()
assert not executor.is_failed
ranks = sorted(h.rank for h in executor.ray_worker_handles)
assert ranks == list(range(world_size))
for handle in executor.ray_worker_handles:
assert handle.node_id is not None
def _detached_rank0_worker() -> ray_executor_v2.RayWorkerProc:
"""A RayWorkerProc that can create its dist-init store without Ray."""
worker = ray_executor_v2.RayWorkerProc.__new__(ray_executor_v2.RayWorkerProc)
worker._parallel_config = SimpleNamespace(world_size=1)
return worker
def _assert_port_held(port: int):
with socket.socket() as contender, pytest.raises(OSError) as exc_info:
contender.bind(("127.0.0.1", port))
assert exc_info.value.errno == errno.EADDRINUSE
def test_dist_init_store_owns_port(monkeypatch):
"""The rank-0 actor keeps the selected TCPStore port bound."""
monkeypatch.setattr(ray.util, "get_node_ip_address", lambda: "127.0.0.1")
worker = _detached_rank0_worker()
init_method = worker.create_dist_init_method()
port = urlsplit(init_method).port
assert port is not None and port != 0
_assert_port_held(port)
reused_store = TCPStore(
"127.0.0.1",
port,
world_size=1,
is_master=True,
wait_for_workers=False,
multi_tenant=True,
)
worker._dist_init_store.set("key", "value")
assert reused_store.get("key") == b"value"
def test_colocated_dist_init_stores_hold_distinct_ports(monkeypatch):
"""Replicas sharing a host must never publish the same TCPStore port.
Each rank-0 store binds a kernel-assigned ephemeral port and keeps it
for the actor's lifetime, so concurrently live replicas cannot collide.
"""
monkeypatch.setattr(ray.util, "get_node_ip_address", lambda: "127.0.0.1")
workers = [_detached_rank0_worker() for _ in range(3)]
ports = [urlsplit(w.create_dist_init_method()).port for w in workers]
assert len(set(ports)) == len(workers)
for worker, port in zip(workers, ports):
assert port == worker._dist_init_store.port
assert port >= 1024
_assert_port_held(port)
@pytest.mark.parametrize("tp_size, pp_size", [(1, 1), (2, 1), (4, 1), (2, 2)])
def test_ray_v2_executor(tp_size, pp_size):
"""Validate RayExecutorV2 with various TP/PP configs."""
vllm_config = create_vllm_config(
tensor_parallel_size=tp_size,
pipeline_parallel_size=pp_size,
)
executor = RayExecutorV2(vllm_config=vllm_config)
try:
assert_executor(executor, tp_size, pp_size)
finally:
executor.shutdown()
@pytest.mark.parametrize(
"tp_size, pp_size, create_placement_group",
[(2, 1, 2), (4, 1, 4), (2, 2, 4)],
indirect=["create_placement_group"],
)
def test_ray_v2_executor_pg(tp_size, pp_size, create_placement_group):
"""Validate RayExecutorV2 with various TP/PP configs using external PG."""
vllm_config = create_vllm_config(
tensor_parallel_size=tp_size,
pipeline_parallel_size=pp_size,
placement_group=create_placement_group,
)
executor = RayExecutorV2(vllm_config=vllm_config)
try:
assert_executor(executor, tp_size, pp_size)
finally:
executor.shutdown()
@pytest.mark.parametrize(
"executor",
[create_vllm_config(tensor_parallel_size=2)],
indirect=True,
)
def test_ray_v2_executor_failure_callback(executor):
"""Validate failure callback registration."""
callback_invoked = False
def test_callback():
nonlocal callback_invoked
callback_invoked = True
executor.register_failure_callback(test_callback)
assert not callback_invoked
executor.is_failed = True
executor.register_failure_callback(test_callback)
assert callback_invoked
@pytest.mark.parametrize(
"executor",
[create_vllm_config(tensor_parallel_size=2)],
indirect=True,
)
def test_ray_v2_executor_collective_rpc(executor):
"""Validate collective RPC calls through MessageQueue."""
executor.check_health()
assert not executor.is_failed
assert executor.rpc_broadcast_mq is not None
@pytest.mark.parametrize(
"executor",
[create_vllm_config(tensor_parallel_size=2)],
indirect=True,
)
def test_ray_v2_executor_driver_node_rank_0(executor):
"""Validate that driver node workers get the lowest ranks."""
driver_node = ray.get_runtime_context().get_node_id()
for handle in executor.ray_worker_handles:
assert handle.node_id == driver_node
rank0_handle = next(h for h in executor.ray_worker_handles if h.rank == 0)
assert rank0_handle.node_id == driver_node
@pytest.mark.parametrize(
"executor",
[create_vllm_config(tensor_parallel_size=2)],
indirect=True,
)
def test_ray_v2_executor_worker_death(executor):
"""Validate executor detects worker death via ray.wait()."""
callback_event = threading.Event()
def on_failure():
callback_event.set()
executor.register_failure_callback(on_failure)
assert not executor.is_failed
# Kill one worker actor externally
victim = executor.ray_worker_handles[1].actor
ray.kill(victim, no_restart=True)
# Monitor thread should detect the death and invoke callback
assert callback_event.wait(timeout=30)
assert executor.is_failed
assert executor.shutting_down
def test_ray_v2_executor_shutdown():
"""Validate graceful shutdown: ray.kill() terminates all worker actors."""
executor = RayExecutorV2(vllm_config=create_vllm_config(tensor_parallel_size=2))
assert executor.rpc_broadcast_mq is not None
assert len(executor.response_mqs) == executor.world_size
actors = [h.actor for h in executor.ray_worker_handles]
executor.shutdown()
for actor in actors:
with pytest.raises(ray.exceptions.RayActorError):
ray.get(actor.wait_for_init.remote(), timeout=5)
assert executor.rpc_broadcast_mq is None
assert len(executor.response_mqs) == 0
@pytest.mark.parametrize(
"executor",
[create_vllm_config(tensor_parallel_size=2)],
indirect=True,
)
def test_ray_v2_run_refs_stored_for_monitoring(executor):
"""Validate worker handles store run_ref for monitoring."""
for handle in executor.ray_worker_handles:
assert handle.run_ref is not None
ready, _ = ray.wait([handle.run_ref], timeout=0)
assert len(ready) == 0, "run_ref should be pending"
@pytest.mark.parametrize("tp_size, pp_size", [(2, 1), (2, 2)])
def test_ray_v2_single_node_generation(tp_size, pp_size):
"""End-to-end LLM generation with RayExecutorV2."""
llm = LLM(
model=MODEL,
tensor_parallel_size=tp_size,
pipeline_parallel_size=pp_size,
distributed_executor_backend="ray",
enforce_eager=True,
max_model_len=256,
gpu_memory_utilization=0.3,
)
try:
prompts = [
"Hello, my name is",
"The capital of France is",
"The future of AI is",
]
outputs = llm.generate(prompts)
assert len(outputs) == len(prompts)
for output in outputs:
assert len(output.outputs) > 0
assert len(output.outputs[0].text) > 0
finally:
llm.llm_engine.model_executor.shutdown()
del llm
gc.collect()
@pytest.mark.parametrize(
"bundle_indices, expected_bundle_ids, create_placement_group",
[("2,3", [2, 3], 4), ("3,2", [3, 2], 4)],
indirect=["create_placement_group"],
)
def test_ray_v2_bundle_indices_env(
bundle_indices, expected_bundle_ids, create_placement_group, monkeypatch
):
"""Validate explicit VLLM_RAY_BUNDLE_INDICES bundle placement."""
monkeypatch.setenv("VLLM_RAY_BUNDLE_INDICES", bundle_indices)
vllm_config = create_vllm_config(
tensor_parallel_size=2,
placement_group=create_placement_group,
)
executor = RayExecutorV2(vllm_config=vllm_config)
try:
actual = [
h.bundle_id_idx
for h in sorted(executor.ray_worker_handles, key=lambda h: h.rank)
]
assert actual == expected_bundle_ids
assert_executor(executor, tp_size=2, pp_size=1)
finally:
executor.shutdown()
@pytest.mark.parametrize(
"bundle_indices, expected_error, create_placement_group",
[
("1,1", "cannot have duplicate values,", 4),
("0,1,2", "must have the same size", 4),
],
indirect=["create_placement_group"],
)
def test_ray_v2_invalid_bundle_indices(
bundle_indices, expected_error, create_placement_group, monkeypatch
):
"""Validate invalid bundle indices are rejected."""
monkeypatch.setenv("VLLM_RAY_BUNDLE_INDICES", bundle_indices)
vllm_config = create_vllm_config(
tensor_parallel_size=2, placement_group=create_placement_group
)
with pytest.raises(AssertionError, match=expected_error):
RayExecutorV2(vllm_config=vllm_config)
@pytest.mark.parametrize("tp_size, pp_size", [(2, 1), (2, 2)])
def test_ray_v2_single_node_generation_with_pg(tp_size, pp_size):
"""E2E LLM generation with a user-provided placement group."""
ensure_ray_initialized()
bundles = [{"GPU": 1, "CPU": 1} for _ in range(tp_size * pp_size)]
pg = ray.util.placement_group(bundles, strategy="PACK")
ray.get(pg.ready())
try:
with patch.object(ray.util, "get_current_placement_group", return_value=pg):
llm = LLM(
model=MODEL,
tensor_parallel_size=tp_size,
pipeline_parallel_size=pp_size,
distributed_executor_backend="ray",
enforce_eager=True,
max_model_len=256,
gpu_memory_utilization=0.3,
)
prompts = [
"Hello, my name is",
"The capital of France is",
"The future of AI is",
]
outputs = llm.generate(prompts)
assert len(outputs) == len(prompts)
for output in outputs:
assert len(output.outputs) > 0
assert len(output.outputs[0].text) > 0
finally:
llm.llm_engine.model_executor.shutdown()
del llm
gc.collect()
@@ -1,304 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import ray
import torch
import torch.distributed as dist
from vllm._aiter_ops import is_aiter_found, rocm_aiter_ops
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
from vllm.distributed.parallel_state import get_dp_group, get_tp_group, graph_capture
from vllm.envs import disable_envs_cache
from vllm.platforms import current_platform
from ..utils import (
assert_rocm_custom_allreduce_backend_state,
ensure_model_parallel_initialized,
init_test_distributed_environment,
multi_gpu_test,
multi_process_parallel,
)
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm-only AITER custom allreduce tests",
)
test_cases = [
((2, 7168), torch.float16),
((2, 7168), torch.bfloat16),
((128, 8192), torch.float16),
((128, 8192), torch.bfloat16),
]
def _configure_aiter_custom_ar_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising=False)
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1")
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", "NONE")
disable_envs_cache()
rocm_aiter_ops.refresh_env_variables()
def _assert_aiter_handles_input(inp: torch.Tensor) -> None:
aiter_ar_comm = get_tp_group().device_communicator.aiter_ar_comm
assert aiter_ar_comm is not None
assert aiter_ar_comm.should_custom_ar(inp), (
f"AITER CustomAllreduce does not support input shape {inp.shape}."
)
@ray.remote(num_gpus=1, max_calls=1)
def graph_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
) -> None:
with monkeypatch.context() as m:
_configure_aiter_custom_ar_env(m)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
assert_rocm_custom_allreduce_backend_state(True, "NONE")
group = get_tp_group().device_group
# A small all_reduce for warmup.
# this is needed because device communicators might be created lazily
# (e.g. NCCL). This will ensure that the communicator is initialized
# before any communication happens, so that this group can be used for
# graph capture immediately.
data = torch.zeros(1)
data = data.to(device=device)
dist.all_reduce(data, group=group)
torch.accelerator.synchronize()
del data
for shape, dtype in test_cases:
with graph_capture(device=device) as graph_capture_context:
inp = torch.ones(shape, dtype=dtype, device=device)
_assert_aiter_handles_input(inp)
expected = inp * tp_size
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
out = tensor_model_parallel_all_reduce(inp)
graph.replay()
torch.testing.assert_close(out, expected)
@ray.remote(num_gpus=1, max_calls=1)
def eager_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
) -> None:
with monkeypatch.context() as m:
_configure_aiter_custom_ar_env(m)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
assert_rocm_custom_allreduce_backend_state(True, "NONE")
for shape, dtype in test_cases:
inp = torch.ones(shape, dtype=dtype, device=device)
_assert_aiter_handles_input(inp)
expected = inp * tp_size
out = tensor_model_parallel_all_reduce(inp)
torch.testing.assert_close(out, expected)
@pytest.mark.skipif(not is_aiter_found(), reason="AITER is not installed")
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("tp_size", [2])
@pytest.mark.parametrize("pipeline_parallel_size", [1])
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
def test_rocm_aiter_custom_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pipeline_parallel_size,
test_target,
):
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
def _get_aiter_ag_rs_comm():
device_communicator = get_dp_group().device_communicator
assert device_communicator.use_aiter_ag_rs, (
"AITER custom AG/RS was not enabled on the DP group."
)
aiter_comm = device_communicator.aiter_ar_comm
assert aiter_comm is not None, "AITER custom AG/RS was not initialized."
assert not aiter_comm.disabled, "AITER custom AG/RS is disabled."
return aiter_comm
def _assert_aiter_handles_ag(aiter_comm, inp: torch.Tensor) -> None:
assert aiter_comm.should_custom_ag(inp), (
f"AITER custom all-gather does not support input shape {inp.shape}."
)
def _assert_aiter_handles_rs(aiter_comm, inp: torch.Tensor) -> None:
assert aiter_comm.should_custom_rs(inp, dim=0), (
f"AITER custom reduce-scatter does not support input shape {inp.shape}."
)
@ray.remote(num_gpus=1, max_calls=1)
def eager_ag_rs(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
data_parallel_size,
data_parallel_master_port,
) -> None:
with monkeypatch.context() as m:
_configure_aiter_custom_ar_env(m)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(
tp_size,
pp_size,
rank,
distributed_init_port,
data_parallel_size=data_parallel_size,
data_parallel_master_port=data_parallel_master_port,
)
dp_group = get_dp_group()
group = dp_group.device_group
dp_world = dp_group.world_size
aiter_comm = _get_aiter_ag_rs_comm()
for shape, dtype in test_cases:
num_tokens, hidden = shape
# all-gather: each rank contributes (num_tokens, hidden).
inp = torch.ones(shape, dtype=dtype, device=device) * (rank + 1)
_assert_aiter_handles_ag(aiter_comm, inp)
expected = torch.empty(
(num_tokens * dp_world, hidden), dtype=dtype, device=device
)
dist.all_gather_into_tensor(expected, inp, group=group)
out = aiter_comm.custom_all_gather(inp, dim=0)
assert out is not None
torch.testing.assert_close(out, expected)
# reduce-scatter: each rank contributes (num_tokens * dp, hidden).
rs_in = torch.ones(
(num_tokens * dp_world, hidden), dtype=dtype, device=device
) * (rank + 1)
_assert_aiter_handles_rs(aiter_comm, rs_in)
rs_expected = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
dist.reduce_scatter_tensor(rs_expected, rs_in, group=group)
rs_out = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
aiter_comm.custom_reduce_scatter(rs_in, rs_out, dim=0)
torch.testing.assert_close(rs_out, rs_expected)
@ray.remote(num_gpus=1, max_calls=1)
def graph_ag_rs(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
data_parallel_size,
data_parallel_master_port,
) -> None:
with monkeypatch.context() as m:
_configure_aiter_custom_ar_env(m)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(
tp_size,
pp_size,
rank,
distributed_init_port,
data_parallel_size=data_parallel_size,
data_parallel_master_port=data_parallel_master_port,
)
dp_group = get_dp_group()
group = dp_group.device_group
dp_world = dp_group.world_size
aiter_comm = _get_aiter_ag_rs_comm()
# Warmup so DP comms is initialized before graph capture
data = torch.zeros(1, device=device)
dist.all_reduce(data, group=group)
torch.accelerator.synchronize()
del data
for shape, dtype in test_cases:
num_tokens, hidden = shape
# all-gather under graph capture.
inp = torch.ones(shape, dtype=dtype, device=device) * (rank + 1)
_assert_aiter_handles_ag(aiter_comm, inp)
ag_expected = torch.empty(
(num_tokens * dp_world, hidden), dtype=dtype, device=device
)
dist.all_gather_into_tensor(ag_expected, inp, group=group)
with graph_capture(device=device) as graph_capture_context:
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
ag_out = aiter_comm.custom_all_gather(inp, dim=0)
graph.replay()
torch.testing.assert_close(ag_out, ag_expected)
# reduce-scatter under graph capture.
rs_in = torch.ones(
(num_tokens * dp_world, hidden), dtype=dtype, device=device
) * (rank + 1)
_assert_aiter_handles_rs(aiter_comm, rs_in)
rs_expected = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
dist.reduce_scatter_tensor(rs_expected, rs_in, group=group)
rs_out = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
with graph_capture(device=device) as graph_capture_context:
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
aiter_comm.custom_reduce_scatter(rs_in, rs_out, dim=0)
graph.replay()
torch.testing.assert_close(rs_out, rs_expected)
@pytest.mark.skipif(not is_aiter_found(), reason="AITER is not installed")
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("tp_size", [1])
@pytest.mark.parametrize("pipeline_parallel_size", [1])
@pytest.mark.parametrize("data_parallel_size", [2])
@pytest.mark.parametrize("test_target", [eager_ag_rs, graph_ag_rs])
def test_rocm_aiter_custom_ag_rs(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pipeline_parallel_size,
data_parallel_size,
test_target,
):
multi_process_parallel(
monkeypatch,
tp_size,
pipeline_parallel_size,
test_target,
data_parallel_size=data_parallel_size,
)
-884
View File
@@ -1,884 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import importlib
import multiprocessing as mp
import os
import queue
import traceback
from functools import lru_cache
from types import SimpleNamespace
from typing import Literal
from unittest.mock import patch
import pytest
import torch
import torch.distributed as dist
import vllm.envs as envs
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
from vllm.utils.network_utils import get_open_port
from ..utils import multi_gpu_test
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm-only quick-reduce tests",
)
MB = 1024 * 1024
WORLD_SIZE = 2
QUANT_LEVELS = ["FP", "INT8", "INT6", "INT4"]
def _log(message: str) -> None:
print(f"[rocm_quick_reduce] {message}", flush=True)
def _reload_envs():
return importlib.reload(envs)
def _make_quick_allreduce(
*,
disabled: bool = False,
world_size: int = 2,
quant_level: str = "FP",
use_fp16_kernels: bool = False,
qr_max_size: int = 64 * MB,
):
from vllm.distributed.device_communicators.quick_all_reduce import (
QuickAllReduce,
QuickReduceRegime,
)
qar = QuickAllReduce.__new__(QuickAllReduce)
qar.disabled = disabled
qar.world_size = world_size
qar.use_fp16_kernels = use_fp16_kernels
qar.qr_quant_level = QuickReduceRegime[quant_level]
qar.qr_max_size = qr_max_size
qar.qr_min_size = None
qar.qr_quantization_min_size = None
return qar
def _quick_allreduce_worker(
rank: int,
port: int,
quant_level: str,
dtype_name: str,
cast_bf16: bool,
):
os.environ["VLLM_ROCM_QUICK_REDUCE_QUANTIZATION"] = quant_level
os.environ["VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16"] = "1" if cast_bf16 else "0"
_log(
f"worker start: rank={rank} quant={quant_level} "
f"dtype={dtype_name} cast_bf16={cast_bf16}"
)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
dist.init_process_group(
backend="gloo",
init_method=f"tcp://127.0.0.1:{port}",
rank=rank,
world_size=WORLD_SIZE,
)
qar = None
try:
from vllm.distributed.device_communicators.quick_all_reduce import (
QuickAllReduce,
)
qar = QuickAllReduce(group=dist.GroupMember.WORLD, device=rank)
assert not qar.disabled
num_elements = 8 * MB if dtype_name == "float16" else 4 * MB
dtype = getattr(torch, dtype_name)
inp = torch.ones(num_elements, dtype=dtype, device=device)
assert qar.should_quick_allreduce(inp)
if cast_bf16:
assert qar.use_fp16_kernels
out = qar.quick_all_reduce(inp)
assert torch.allclose(out, inp * WORLD_SIZE, atol=2.5, rtol=0.1)
_log(
f"worker complete: rank={rank} quant={quant_level} "
f"dtype={dtype_name} num_elements={num_elements} "
f"use_fp16_kernels={qar.use_fp16_kernels}"
)
finally:
if qar is not None:
qar.close()
if dist.is_initialized():
dist.destroy_process_group()
def _run_two_gpu_quick_allreduce_test(
*,
quant_level: str,
dtype_name: str,
cast_bf16: bool,
):
_log(
f"launch 2-GPU case: quant={quant_level} "
f"dtype={dtype_name} cast_bf16={cast_bf16}"
)
ctx = mp.get_context("spawn")
port = get_open_port()
procs = []
for rank in range(WORLD_SIZE):
proc = ctx.Process(
target=_quick_allreduce_worker,
args=(rank, port, quant_level, dtype_name, cast_bf16),
)
proc.start()
procs.append(proc)
for proc in procs:
proc.join(timeout=60)
assert proc.exitcode == 0, f"worker exited with code {proc.exitcode}"
_log(
f"finished 2-GPU case: quant={quant_level} "
f"dtype={dtype_name} cast_bf16={cast_bf16}"
)
CUDAGRAPH_WORLD_SIZE = 2
CUDAGRAPH_ROUNDS = 10
CUDAGRAPH_NUM_ELEMENTS = 1 << 21 # 2M fp16 = 4 MB, above the quick-reduce thresholds
def _quick_allreduce_cudagraph_worker(
rank: int,
world_size: int,
port: int,
quant_level: str,
):
# FP keeps the all-reduce bit-exact for small fp16 integers, so every
# replayed round can be checked exactly.
os.environ["VLLM_ROCM_QUICK_REDUCE_QUANTIZATION"] = quant_level
os.environ["VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16"] = "0"
_log(f"cudagraph worker start: rank={rank} quant={quant_level}")
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
dist.init_process_group(
backend="gloo",
init_method=f"tcp://127.0.0.1:{port}",
rank=rank,
world_size=world_size,
)
qar = None
try:
from vllm.distributed.device_communicators.quick_all_reduce import (
QuickAllReduce,
)
qar = QuickAllReduce(group=dist.GroupMember.WORLD, device=rank)
assert not qar.disabled
N = CUDAGRAPH_NUM_ELEMENTS
inp = torch.empty(N, dtype=torch.float16, device=device)
out = torch.empty(N, dtype=torch.float16, device=device)
assert qar.should_quick_allreduce(inp)
# Every rank contributes the same value v in a round, so the true
# cross-rank all-reduce sum is simply world_size * v.
def expected(v):
return float(world_size * v)
if rank == 0:
print(
f"[repro] world_size={world_size} elems={N} regime={quant_level} fp16",
flush=True,
)
# Warmup, then capture a graph with EXACTLY ONE quick-reduce (isolated
# qr, so it is the sole writer of its flag slot -- the condition that
# triggers the stale-flag bug).
inp.fill_(1.0)
qar.quick_all_reduce(inp, out=out)
torch.accelerator.synchronize()
dist.barrier()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
qar.quick_all_reduce(inp, out=out)
torch.accelerator.synchronize()
dist.barrier()
for v in range(CUDAGRAPH_ROUNDS):
inp.fill_(float(v)) # in-place: same value on every rank
dist.barrier()
g.replay()
torch.accelerator.synchronize()
dist.barrier()
got = out.float()
expect = expected(v)
if rank == 0:
print(f"round {v}: got={got[:10]}, expected={expect}", flush=True)
mismatch = ~torch.isclose(got, torch.full_like(got, expect))
num_mismatch = int(mismatch.sum().item())
assert num_mismatch == 0, (
f"rank={rank} round={v} expected={expect} "
f"mismatched {num_mismatch}/{got.numel()} elements; "
f"unique wrong values={torch.unique(got[mismatch])[:8].tolist()}"
)
_log(f"cudagraph worker complete: rank={rank} rounds={CUDAGRAPH_ROUNDS}")
finally:
if qar is not None:
qar.close()
if dist.is_initialized():
dist.destroy_process_group()
def _run_cudagraph_replay_test(*, world_size: int, quant_level: str):
_log(f"launch {world_size}-GPU cudagraph replay case: quant={quant_level}")
ctx = mp.get_context("spawn")
port = get_open_port()
procs = []
for rank in range(world_size):
proc = ctx.Process(
target=_quick_allreduce_cudagraph_worker,
args=(rank, world_size, port, quant_level),
)
proc.start()
procs.append(proc)
for proc in procs:
proc.join(timeout=120)
assert proc.exitcode == 0, f"worker exited with code {proc.exitcode}"
_log(f"finished {world_size}-GPU cudagraph replay case: quant={quant_level}")
MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct"
E2E_PREFILL_TOKENS = 1024
E2E_MAX_MODEL_LEN = 1536
E2E_GPU_MEMORY_UTILIZATION = 0.3
E2E_KV_CACHE_MEMORY_BYTES = 2 << 30
_BACKGROUND_LINE = (
"Background filler: this archived operations memo repeats a routine status "
"line so the distributed test uses a realistically long prefill."
)
_BACKGROUND_BLOCK = " ".join([_BACKGROUND_LINE] * 48)
def _build_prompt(*, fact_block: str, question: str) -> str:
return (
"Read the archived operations memo below. Most of the memo is filler. "
"Use only the fact block near the end when answering.\n"
f"{_BACKGROUND_BLOCK}\n"
"Fact block:\n"
f"{fact_block}\n"
f"Question: {question}\n"
"Answer in one short sentence."
)
E2E_PROMPTS = [
_build_prompt(
fact_block=(
"- Festival city: Oslo\n- Mascot animal: otter\n- Welcome drink: tea"
),
question="Which city hosts the festival, and what animal is the mascot?",
),
_build_prompt(
fact_block=(
"- Meeting day: Tuesday\n"
"- Planned snack: apricot cake\n"
"- Backup room: Cedar"
),
question="What day is the meeting, and what snack is planned?",
),
]
RECORDED_RESPONSE_TEXTS = (
" The city hosting the festival is Oslo, and the mascot is an otter.",
" The meeting is on Tuesday and the snack planned is apricot cake.",
)
REQUIRED_WORDS = (("oslo", "otter"), ("tuesday", "apricot"))
def _log_prompt_summaries() -> None:
for i, prompt in enumerate(E2E_PROMPTS):
prompt_lines = prompt.splitlines()
fact_block = [line for line in prompt_lines if line.startswith("- ")]
fact_summary = "; ".join(line.removeprefix("- ") for line in fact_block)
_log(f"prompt {i} facts: {fact_summary}")
@lru_cache(maxsize=1)
def _get_model_path() -> str:
try:
path = hf_api().snapshot_download(repo_id=MODEL_NAME, local_files_only=True)
_log(f"using cached model snapshot: {path}")
return path
except Exception:
path = hf_api().snapshot_download(repo_id=MODEL_NAME)
_log(f"downloaded model snapshot: {path}")
return path
def _get_hidden_size(model_config) -> int:
hidden_size = getattr(model_config, "hidden_size", None)
if hidden_size is None and hasattr(model_config, "text_config"):
hidden_size = getattr(model_config.text_config, "hidden_size", None)
assert isinstance(hidden_size, int)
return hidden_size
def _check_tp_allreduce_uses_quick_reduce(
self,
num_tokens: int,
dtype_name: str = "float16",
) -> dict[str, int | bool]:
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
from vllm.distributed.parallel_state import get_tp_group
assert self.device is not None
qr_comm = get_tp_group().device_communicator.qr_comm
assert qr_comm is not None
assert not qr_comm.disabled
hidden_size = _get_hidden_size(self.model_runner.model.config)
dtype = getattr(torch, dtype_name)
sample = torch.full(
(num_tokens, hidden_size),
fill_value=float(self.rank + 1),
dtype=dtype,
device=self.device,
)
assert qr_comm.should_quick_allreduce(sample)
expected = sample.clone()
reduced = tensor_model_parallel_all_reduce(sample)
dist.all_reduce(expected, group=get_tp_group().device_group)
torch.testing.assert_close(reduced, expected, atol=2.5, rtol=0.1)
stats = {
"rank": self.rank,
"hidden_size": hidden_size,
"num_tokens": num_tokens,
"use_fp16_kernels": qr_comm.use_fp16_kernels,
}
_log(
"worker quick-reduce check: "
f"rank={self.rank} hidden_size={hidden_size} "
f"num_tokens={num_tokens} use_fp16_kernels={qr_comm.use_fp16_kernels}"
)
return stats
def _check_quick_reduce_disabled(self) -> int:
from vllm.distributed.parallel_state import get_tp_group
qr_comm = get_tp_group().device_communicator.qr_comm
assert qr_comm is not None
assert qr_comm.disabled
_log(f"worker confirmed quick reduce is disabled: rank={self.rank}")
return self.rank
def _collect_generations(outputs) -> list[tuple[tuple[int, ...], str]]:
return [
(tuple(output.outputs[0].token_ids), output.outputs[0].text)
for output in outputs
]
def _shutdown_llm(llm: LLM | None) -> None:
if llm is None:
cleanup_dist_env_and_memory()
return
with contextlib.suppress(Exception):
llm.llm_engine.engine_core.shutdown()
del llm
cleanup_dist_env_and_memory()
def _log_generations(
label: str,
generations: list[tuple[tuple[int, ...], str]],
) -> None:
for i, (token_ids, text) in enumerate(generations):
_log(f"{label} prompt {i} token ids: {list(token_ids)}")
_log(f"{label} prompt {i} text: {text!r}")
def _assert_required_words(
label: str,
generations: list[tuple[tuple[int, ...], str]],
) -> None:
for i, (_, text) in enumerate(generations):
lowered = text.lower()
missing = [word for word in REQUIRED_WORDS[i] if word not in lowered]
assert not missing, (
f"{label} prompt {i} is missing required words {missing}. "
f"Observed text: {text!r}"
)
def _collect_soft_mismatches(
baseline_generations: list[tuple[tuple[int, ...], str]],
quick_reduce_generations: list[tuple[tuple[int, ...], str]],
) -> list[str]:
mismatches = []
for i, (_, text) in enumerate(baseline_generations):
expected = RECORDED_RESPONSE_TEXTS[i]
if text != expected:
mismatches.append(
f"baseline prompt {i} drifted from the recorded response.\n"
f"expected={expected!r}\nactual={text!r}"
)
for i, (_, text) in enumerate(quick_reduce_generations):
expected = RECORDED_RESPONSE_TEXTS[i]
if text != expected:
mismatches.append(
f"quick-reduce prompt {i} drifted from the recorded response.\n"
f"expected={expected!r}\nactual={text!r}"
)
for i, ((_, baseline_text), (_, quick_reduce_text)) in enumerate(
zip(baseline_generations, quick_reduce_generations)
):
if baseline_text != quick_reduce_text:
mismatches.append(
f"baseline and quick-reduce responses differ for prompt {i}.\n"
f"baseline={baseline_text!r}\nquick_reduce={quick_reduce_text!r}"
)
return mismatches
def _run_generation(
*,
backend: Literal["mp", "ray"],
quant_mode: str,
expect_quick_reduce: bool,
) -> list[tuple[tuple[int, ...], str]]:
llm = None
monkeypatch = pytest.MonkeyPatch()
with monkeypatch.context() as m:
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
m.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", quant_mode)
model_path = _get_model_path()
_log(
f"starting generation: backend={backend} quant={quant_mode} "
f"gpu_memory_utilization={E2E_GPU_MEMORY_UTILIZATION} "
f"kv_cache_bytes={E2E_KV_CACHE_MEMORY_BYTES} model={model_path}"
)
try:
llm = LLM(
model=model_path,
tokenizer=model_path,
tensor_parallel_size=2,
distributed_executor_backend=backend,
dtype="half",
enforce_eager=True,
max_model_len=E2E_MAX_MODEL_LEN,
max_num_seqs=len(E2E_PROMPTS),
gpu_memory_utilization=E2E_GPU_MEMORY_UTILIZATION,
kv_cache_memory_bytes=E2E_KV_CACHE_MEMORY_BYTES,
seed=0,
)
if not expect_quick_reduce:
assert llm.collective_rpc(_check_quick_reduce_disabled) == [0, 1]
if expect_quick_reduce:
worker_stats = llm.collective_rpc(
_check_tp_allreduce_uses_quick_reduce,
args=(E2E_PREFILL_TOKENS,),
)
assert [stat["rank"] for stat in worker_stats] == [0, 1]
worker_summary = "; ".join(
"rank={rank} hidden_size={hidden_size} num_tokens={num_tokens} "
"use_fp16_kernels={use_fp16_kernels}".format(**stat)
for stat in worker_stats
)
_log(f"{backend} quick-reduce worker checks: {worker_summary}")
outputs = llm.generate(
E2E_PROMPTS,
SamplingParams(
temperature=0.0,
max_tokens=20,
stop=["\nAnswer:", " Answer:"],
),
use_tqdm=False,
)
generations = _collect_generations(outputs)
assert all(text.strip() for _, text in generations)
_log_generations(f"{backend} {quant_mode}", generations)
return generations
finally:
_shutdown_llm(llm)
def _run_quick_reduce_llm_e2e_in_subprocess(
*,
backend: Literal["mp", "ray"],
) -> str | None:
_log(f"running LLM e2e: backend={backend}")
_log_prompt_summaries()
baseline_outputs = _run_generation(
backend=backend,
quant_mode="NONE",
expect_quick_reduce=False,
)
quick_reduce_outputs = _run_generation(
backend=backend,
quant_mode="FP",
expect_quick_reduce=True,
)
_assert_required_words("baseline", baseline_outputs)
_assert_required_words("quick-reduce", quick_reduce_outputs)
mismatches = _collect_soft_mismatches(baseline_outputs, quick_reduce_outputs)
if mismatches:
details = "\n\n".join(mismatches)
_log(f"soft response mismatch:\n{details}")
return details
_log(f"LLM e2e backend={backend} matched the recorded responses exactly")
return None
def _quick_reduce_llm_e2e_worker(
result_queue: mp.Queue,
backend: Literal["mp", "ray"],
) -> None:
try:
xfail_reason = _run_quick_reduce_llm_e2e_in_subprocess(backend=backend)
except Exception:
result_queue.put({"status": "error", "reason": traceback.format_exc()})
raise
else:
if xfail_reason is not None:
result_queue.put({"status": "xfail", "reason": xfail_reason})
else:
result_queue.put({"status": "ok"})
def run_quick_reduce_llm_e2e(
*,
backend: Literal["mp", "ray"],
) -> None:
ctx = mp.get_context("spawn")
result_queue = ctx.Queue()
proc = ctx.Process(
target=_quick_reduce_llm_e2e_worker,
args=(result_queue, backend),
)
proc.start()
proc.join(timeout=600)
try:
result = result_queue.get(timeout=5)
except queue.Empty as exc:
if proc.exitcode != 0:
raise AssertionError(
f"quick-reduce llm e2e subprocess failed for backend={backend} "
f"with exit code {proc.exitcode} and produced no result"
) from exc
raise AssertionError(
f"quick-reduce llm e2e subprocess produced no result for backend={backend}"
) from exc
if result["status"] == "xfail":
pytest.xfail(result["reason"])
if result["status"] == "error":
raise AssertionError(
f"quick-reduce llm e2e subprocess failed for backend={backend}:\n"
f"{result['reason']}"
)
assert proc.exitcode == 0, (
f"quick-reduce llm e2e subprocess failed for backend={backend} "
f"with exit code {proc.exitcode}"
)
def test_quick_reduce_regime_values():
from vllm.distributed.device_communicators.quick_all_reduce import QuickReduceRegime
assert QuickReduceRegime.FP.value == 0
assert QuickReduceRegime.INT8.value == 1
assert QuickReduceRegime.INT6.value == 2
assert QuickReduceRegime.INT4.value == 3
assert QuickReduceRegime.INT3.value == 4
assert QuickReduceRegime.NONE.value == 5
def test_quick_reduce_regime_names():
from vllm.distributed.device_communicators.quick_all_reduce import QuickReduceRegime
assert set(QuickReduceRegime.__members__) == {
"FP",
"INT8",
"INT6",
"INT4",
"INT3",
"NONE",
}
@pytest.mark.parametrize("quant_level", QUANT_LEVELS + ["NONE"])
def test_quick_reduce_quantization_env_var(monkeypatch, quant_level):
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", quant_level)
reloaded_envs = _reload_envs()
assert quant_level == reloaded_envs.VLLM_ROCM_QUICK_REDUCE_QUANTIZATION
def test_quick_reduce_quantization_default(monkeypatch):
monkeypatch.delenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", raising=False)
reloaded_envs = _reload_envs()
assert reloaded_envs.VLLM_ROCM_QUICK_REDUCE_QUANTIZATION == "NONE"
@pytest.mark.parametrize("cast_bf16", [True, False])
def test_quick_reduce_cast_bf16_to_fp16_env_var(monkeypatch, cast_bf16):
monkeypatch.setenv(
"VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", "1" if cast_bf16 else "0"
)
reloaded_envs = _reload_envs()
assert reloaded_envs.VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16 is cast_bf16
def test_quick_reduce_cast_bf16_to_fp16_default(monkeypatch):
monkeypatch.delenv("VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", raising=False)
reloaded_envs = _reload_envs()
assert reloaded_envs.VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16 is True
@pytest.mark.parametrize("max_mb", [128, 512, 2048, None])
def test_quick_reduce_max_size_env_var(monkeypatch, max_mb):
if max_mb is None:
monkeypatch.delenv("VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", raising=False)
else:
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", str(max_mb))
reloaded_envs = _reload_envs()
assert max_mb == reloaded_envs.VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB
def test_quick_reduce_max_size_default(monkeypatch):
monkeypatch.delenv("VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", raising=False)
reloaded_envs = _reload_envs()
assert reloaded_envs.VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB is None
@pytest.mark.parametrize(
("gcn_arch_name", "expected"),
[
("gfx942", True),
("gfx950", True),
("gfx90a", False),
("", False),
],
)
def test_quick_allreduce_rocm_arch_available(gcn_arch_name, expected):
from vllm.distributed.device_communicators.quick_all_reduce import QuickAllReduce
qar = QuickAllReduce.__new__(QuickAllReduce)
qar.disabled = True
with (
patch(
"vllm.distributed.device_communicators.quick_all_reduce.current_platform."
"is_rocm",
return_value=True,
),
patch(
"torch.cuda.get_device_properties",
return_value=SimpleNamespace(gcnArchName=gcn_arch_name),
),
):
assert qar._rocm_arch_available() is expected
def test_quick_allreduce_rocm_arch_available_handles_probe_failure():
from vllm.distributed.device_communicators.quick_all_reduce import QuickAllReduce
qar = QuickAllReduce.__new__(QuickAllReduce)
qar.disabled = True
with (
patch(
"vllm.distributed.device_communicators.quick_all_reduce.current_platform."
"is_rocm",
return_value=True,
),
patch("torch.cuda.get_device_properties", side_effect=RuntimeError),
):
assert qar._rocm_arch_available() is False
def test_quick_allreduce_rejects_disabled():
qar = _make_quick_allreduce(disabled=True)
inp = torch.zeros(1024, dtype=torch.float16)
assert qar.should_quick_allreduce(inp) is False
def test_quick_allreduce_rejects_unsupported_dtype():
qar = _make_quick_allreduce()
inp = torch.zeros(1024 * 1024, dtype=torch.float32)
assert qar.should_quick_allreduce(inp) is False
def test_quick_allreduce_rejects_non_aligned_input():
qar = _make_quick_allreduce()
inp = torch.zeros(5, dtype=torch.float16)
assert qar.should_quick_allreduce(inp) is False
def test_quick_allreduce_rejects_non_contiguous_input():
qar = _make_quick_allreduce()
inp = torch.zeros((1024, 1024), dtype=torch.float16)[:, ::2]
assert qar.should_quick_allreduce(inp) is False
def test_quick_allreduce_rejects_input_smaller_than_threshold():
qar = _make_quick_allreduce()
inp = torch.zeros((MB // 2) - 8, dtype=torch.float16)
assert qar.should_quick_allreduce(inp) is False
def test_quick_allreduce_accepts_input_at_threshold():
qar = _make_quick_allreduce()
inp = torch.zeros(MB // 2, dtype=torch.float16)
assert qar.should_quick_allreduce(inp) is True
def test_quick_allreduce_rejects_input_larger_than_max_size():
qar = _make_quick_allreduce(qr_max_size=1 * MB)
inp = torch.zeros(MB, dtype=torch.float16)
assert qar.should_quick_allreduce(inp) is False
def test_quick_allreduce_bf16_uses_fp16_threshold_when_cast_enabled():
inp = torch.zeros(MB // 2, dtype=torch.bfloat16)
without_cast = _make_quick_allreduce(use_fp16_kernels=False)
with_cast = _make_quick_allreduce(use_fp16_kernels=True)
assert without_cast.should_quick_allreduce(inp) is False
assert with_cast.should_quick_allreduce(inp) is True
def test_quick_allreduce_supported_world_sizes():
from vllm.distributed.device_communicators.quick_all_reduce import QuickAllReduce
assert QuickAllReduce._SUPPORTED_WORLD_SIZES == [2, 4, 8]
def test_quick_allreduce_supported_dtypes():
from vllm.distributed.device_communicators.quick_all_reduce import QuickAllReduce
assert [torch.float16, torch.bfloat16] == QuickAllReduce._SUPPORTED_DTYPES
def test_quick_allreduce_min_size_table():
from vllm.distributed.device_communicators.quick_all_reduce import QuickAllReduce
for dtype in [torch.float16, torch.bfloat16]:
for world_size in QuickAllReduce._SUPPORTED_WORLD_SIZES:
min_sizes = QuickAllReduce._QR_MIN_SIZE[(dtype, world_size)]
assert len(min_sizes) == 5
assert all(size > 0 for size in min_sizes)
def test_qr_max_size():
from vllm import _custom_ops as ops
max_size = ops.qr_max_size()
assert isinstance(max_size, int)
assert max_size > 0
@pytest.mark.skipif(
current_platform.device_count() < WORLD_SIZE,
reason="requires 2 ROCm GPUs",
)
@pytest.mark.parametrize("quant_level", QUANT_LEVELS)
def test_quick_allreduce_two_gpu_correctness(quant_level):
_log(f"two-GPU correctness case: quant={quant_level}")
_run_two_gpu_quick_allreduce_test(
quant_level=quant_level,
dtype_name="float16",
cast_bf16=False,
)
@multi_gpu_test(num_gpus=CUDAGRAPH_WORLD_SIZE)
def test_quick_allreduce_cudagraph_replay():
# Regression test for the stale flag_color bug: a quick-reduce captured in a
# CUDA graph must return correct results on every replay, not the data from
# a previous round.
_log("cudagraph replay case")
_run_cudagraph_replay_test(
world_size=CUDAGRAPH_WORLD_SIZE,
quant_level="FP",
)
@pytest.mark.skipif(
current_platform.device_count() < WORLD_SIZE,
reason="requires 2 ROCm GPUs",
)
def test_quick_allreduce_bf16_cast_mode():
_log("BF16 cast case")
_run_two_gpu_quick_allreduce_test(
quant_level="FP",
dtype_name="bfloat16",
cast_bf16=True,
)
@pytest.mark.skipif(
current_platform.device_count() < WORLD_SIZE,
reason="requires 2 ROCm GPUs",
)
def test_quick_allreduce_llm_e2e():
_log("LLM e2e case: backend=mp")
run_quick_reduce_llm_e2e(backend="mp")
@pytest.mark.skipif(
current_platform.device_count() < WORLD_SIZE,
reason="requires 2 ROCm GPUs",
)
def test_quick_allreduce_llm_e2e_ray():
_log("LLM e2e case: backend=ray")
run_quick_reduce_llm_e2e(backend="ray")
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,140 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import queue
import random
import typing
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
import vllm.envs as envs
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
from vllm.distributed.parallel_state import (
get_tp_group,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.engine.arg_utils import EngineArgs
from vllm.engine.llm_engine import LLMEngine
from vllm.platforms import current_platform
from vllm.utils.system_utils import update_environment_variables
torch.manual_seed(42)
random.seed(44)
test_size_elements = 1024 * 1024
def symm_mem_allreduce_worker(local_rank: int, world_size: int, q: mp.Queue):
monkeypatch = pytest.MonkeyPatch()
config = VllmConfig(parallel_config=ParallelConfig(tensor_parallel_size=world_size))
with monkeypatch.context() as m, set_current_vllm_config(config):
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
dtype = torch.bfloat16
device = torch.device(f"cuda:{local_rank}")
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.set_default_dtype(dtype)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": "12345",
}
)
init_distributed_environment()
initialize_model_parallel(tensor_model_parallel_size=world_size)
cuda_communicator = typing.cast(
CudaCommunicator, get_tp_group().device_communicator
)
symm_mem_comm = cuda_communicator.symm_mem_comm
if symm_mem_comm is None or symm_mem_comm.disabled:
# can't use skip under multiprocessing
q.put("SymmMemCommunicator is not available or disabled.")
return
inp_direct_symm_mem = torch.randint(
1, 23, (test_size_elements,), dtype=dtype, device=device
)
if not symm_mem_comm.should_use_symm_mem(inp_direct_symm_mem):
# can't use skip under multiprocessing
q.put("SymmMemCommunicator isn't used for this world and input size.")
return
original_inp_direct_symm_mem = inp_direct_symm_mem.clone()
out_direct_symm_mem = symm_mem_comm.all_reduce(inp_direct_symm_mem)
assert out_direct_symm_mem is not None
group = get_tp_group().device_group
dist.all_reduce(original_inp_direct_symm_mem, group=group)
torch.testing.assert_close(
out_direct_symm_mem, original_inp_direct_symm_mem, atol=2.5, rtol=0.1
)
# Test tensor_model_parallel_all_reduce which should use symm_mem
inp_tensor_parallel = torch.randint(
-23, 1, (test_size_elements,), dtype=dtype, device=device
)
original_inp_tensor_parallel = inp_tensor_parallel.clone()
out_tensor_parallel = tensor_model_parallel_all_reduce(inp_tensor_parallel)
dist.all_reduce(original_inp_tensor_parallel, group=group)
torch.testing.assert_close(
out_tensor_parallel, original_inp_tensor_parallel, atol=2.5, rtol=0.1
)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="SymmMemAllreduce is only available for CUDA platforms.",
)
@pytest.mark.parametrize("tp_size", [2])
@pytest.mark.parametrize("pipeline_parallel_size", [1])
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
def test_symm_mem_allreduce(
monkeypatch: pytest.MonkeyPatch, tp_size, pipeline_parallel_size
):
world_size = tp_size * pipeline_parallel_size
if world_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
q = mp.get_context("spawn").Queue()
mp.spawn(symm_mem_allreduce_worker, args=(world_size, q), nprocs=world_size)
try:
val = q.get(timeout=1)
except queue.Empty:
val = None
finally:
cleanup_dist_env_and_memory()
if val is not None:
pytest.skip(val)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="SymmMemAllreduce is only available for CUDA platforms.",
)
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
def test_dp_with_symm_mem_allreduce(monkeypatch: pytest.MonkeyPatch):
world_size = 4
if world_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
# Verify that the DataParallel runs without error
engine_args = EngineArgs(
model="distilbert/distilgpt2",
enforce_eager=True,
enable_prefix_caching=True,
data_parallel_size=2,
tensor_parallel_size=2,
data_parallel_backend="mp",
)
LLMEngine.from_engine_args(engine_args)
@@ -1,82 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# unit test for `examples/features/torchrun/torchrun_example_offline.py`
import os
import random
import torch
import torch.distributed as dist
import vllm.envs as envs
from vllm import LLM, SamplingParams
from vllm.distributed.parallel_state import get_world_group
# By default, let PyTorch choose the WORLD backend for the current device
# type (legacy lazy-init path). When VLLM_DISTRIBUTED_USE_SPLIT_GROUP=1,
# use the explicit eager-init pattern required by `split_group` (mixed
# cpu:gloo,cuda:nccl backend + device_id binding).
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
local_rank = int(os.environ["LOCAL_RANK"])
torch.accelerator.set_device_index(local_rank)
dist.init_process_group(
backend="cpu:gloo,cuda:nccl",
device_id=torch.device(f"cuda:{local_rank}"),
)
else:
dist.init_process_group()
# Create prompts
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
# set different `gpu_memory_utilization` for different ranks,
# to test if all ranks agree on the same kv cache configuration.
llm = LLM(
model="facebook/opt-125m",
tensor_parallel_size=2,
pipeline_parallel_size=int(os.getenv("PP_SIZE", 1)),
distributed_executor_backend="external_launcher",
gpu_memory_utilization=random.uniform(0.8, 0.92),
seed=0,
)
outputs = llm.generate(prompts, sampling_params)
cpu_group = get_world_group().cpu_group
torch_rank = dist.get_rank(group=cpu_group)
def test_consistent_across_ranks(obj):
if torch_rank == 0:
dist.broadcast_object_list([obj], src=0, group=cpu_group)
else:
container = [None]
dist.broadcast_object_list(container, src=0, group=cpu_group)
assert container[0] == obj
test_consistent_across_ranks(llm.llm_engine.vllm_config.cache_config.num_cpu_blocks)
test_consistent_across_ranks(llm.llm_engine.vllm_config.cache_config.num_gpu_blocks)
# make sure we can access the model parameters from the calling process
# of the `LLM` instance.
params = list(
llm.llm_engine.model_executor.driver_worker.worker.model_runner.model.parameters()
)
test_consistent_across_ranks(len(params))
# all ranks should have the same outputs
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
test_consistent_across_ranks(prompt)
test_consistent_across_ranks(generated_text)
print(f"Rank {torch_rank}, Prompt: {prompt!r}, Generated text: {generated_text!r}")
@@ -1,91 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# unit test for `examples/features/torchrun/torchrun_example_offline.py`
import os
import random
import torch
import torch.distributed as dist
import vllm.envs as envs
from vllm import LLM, SamplingParams
from vllm.distributed.parallel_state import get_tp_group, get_world_group
# By default, let PyTorch choose the WORLD backend for the current device
# type (legacy lazy-init path). When VLLM_DISTRIBUTED_USE_SPLIT_GROUP=1,
# use the explicit eager-init pattern required by `split_group` (mixed
# cpu:gloo,cuda:nccl backend + device_id binding).
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
local_rank = int(os.environ["LOCAL_RANK"])
torch.accelerator.set_device_index(local_rank)
dist.init_process_group(
backend="cpu:gloo,cuda:nccl",
device_id=torch.device(f"cuda:{local_rank}"),
)
else:
dist.init_process_group()
# Create prompts
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
] * 10
dp_size = int(os.getenv("DP_SIZE", "1"))
dp_rank = int(os.getenv("DP_RANK", "0"))
if dp_size > 1:
# distribute the prompts across the data parallel ranks
prompts = [prompt for idx, prompt in enumerate(prompts) if idx % dp_size == dp_rank]
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
# set different `gpu_memory_utilization` for different ranks,
# to test if all ranks agree on the same kv cache configuration.
llm = LLM(
model="microsoft/Phi-mini-MoE-instruct",
tensor_parallel_size=int(os.getenv("TP_SIZE", "1")),
pipeline_parallel_size=int(os.getenv("PP_SIZE", "1")),
enable_expert_parallel=int(os.getenv("ENABLE_EP", "0")) == 1,
distributed_executor_backend="external_launcher",
gpu_memory_utilization=random.uniform(0.8, 0.92),
seed=0,
max_model_len=1024,
max_num_seqs=16,
)
outputs = llm.generate(prompts, sampling_params)
group = get_world_group() if dp_size == 1 else get_tp_group()
cpu_group = group.cpu_group
group_rank = dist.get_rank(group=cpu_group)
def test_consistent_across_ranks(obj):
if group_rank == 0:
dist.broadcast_object_list([obj], src=group.ranks[0], group=cpu_group)
else:
container = [None]
dist.broadcast_object_list(container, src=group.ranks[0], group=cpu_group)
assert container[0] == obj
test_consistent_across_ranks(llm.llm_engine.vllm_config.cache_config.num_cpu_blocks)
test_consistent_across_ranks(llm.llm_engine.vllm_config.cache_config.num_gpu_blocks)
# make sure we can access the model parameters from the calling process
# of the `LLM` instance.
params = list(
llm.llm_engine.model_executor.driver_worker.worker.model_runner.model.parameters()
)
test_consistent_across_ranks(len(params))
# all ranks should have the same outputs
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
test_consistent_across_ranks(prompt)
test_consistent_across_ranks(generated_text)
print(f"Rank {group_rank}, Prompt: {prompt!r}, Generated text: {generated_text!r}")
-141
View File
@@ -1,141 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import socket
import pytest
import ray
import torch
import vllm.envs as envs
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
from ..utils import multi_gpu_test
@ray.remote
class _CUDADeviceCountStatelessTestActor:
def get_count(self):
return current_platform.device_count()
def set_cuda_visible_devices(self, cuda_visible_devices: str):
update_environment_variables({"CUDA_VISIBLE_DEVICES": cuda_visible_devices})
def get_cuda_visible_devices(self):
return envs.CUDA_VISIBLE_DEVICES
def test_cuda_device_count_stateless():
"""Test that cuda_device_count_stateless changes return value if
CUDA_VISIBLE_DEVICES is changed."""
if current_platform.is_rocm():
pytest.skip("Skip for ROCm because Ray uses HIP_VISIBLE_DEVICES.")
actor = _CUDADeviceCountStatelessTestActor.options( # type: ignore
num_gpus=2
).remote()
assert len(sorted(ray.get(actor.get_cuda_visible_devices.remote()).split(","))) == 2
assert ray.get(actor.get_count.remote()) == 2
ray.get(actor.set_cuda_visible_devices.remote("0"))
assert ray.get(actor.get_count.remote()) == 1
ray.get(actor.set_cuda_visible_devices.remote(""))
assert ray.get(actor.get_count.remote()) == 0
def cpu_worker(rank, WORLD_SIZE, port1, port2):
pg1 = StatelessProcessGroup.create(
host="127.0.0.1", port=port1, rank=rank, world_size=WORLD_SIZE
)
if rank <= 2:
pg2 = StatelessProcessGroup.create(
host="127.0.0.1", port=port2, rank=rank, world_size=3
)
data = torch.tensor([rank])
data = pg1.broadcast_obj(data, src=2)
assert data.item() == 2
if rank <= 2:
data = torch.tensor([rank + 1])
data = pg2.broadcast_obj(data, src=2)
assert data.item() == 3
pg2.barrier()
pg1.barrier()
def gpu_worker(rank, WORLD_SIZE, port1, port2):
torch.accelerator.set_device_index(rank)
pg1 = StatelessProcessGroup.create(
host="127.0.0.1", port=port1, rank=rank, world_size=WORLD_SIZE
)
pynccl1 = PyNcclCommunicator(pg1, device=rank)
if rank <= 2:
pg2 = StatelessProcessGroup.create(
host="127.0.0.1", port=port2, rank=rank, world_size=3
)
pynccl2 = PyNcclCommunicator(pg2, device=rank)
data = torch.tensor([rank]).cuda()
pynccl1.all_reduce(data)
pg1.barrier()
torch.accelerator.synchronize()
if rank <= 2:
pynccl2.all_reduce(data)
pg2.barrier()
torch.accelerator.synchronize()
item = data[0].item()
print(f"rank: {rank}, item: {item}")
if rank == 3:
assert item == 6
else:
assert item == 18
def broadcast_worker(rank, WORLD_SIZE, port1, port2):
pg1 = StatelessProcessGroup.create(
host="127.0.0.1", port=port1, rank=rank, world_size=WORLD_SIZE
)
if rank == 2:
pg1.broadcast_obj("secret", src=2)
else:
obj = pg1.broadcast_obj(None, src=2)
assert obj == "secret"
pg1.barrier()
def allgather_worker(rank, WORLD_SIZE, port1, port2):
pg1 = StatelessProcessGroup.create(
host="127.0.0.1", port=port1, rank=rank, world_size=WORLD_SIZE
)
data = pg1.all_gather_obj(rank)
assert data == list(range(WORLD_SIZE))
pg1.barrier()
@pytest.mark.skip(reason="This test is flaky and prone to hang.")
@multi_gpu_test(num_gpus=4)
@pytest.mark.parametrize(
"worker", [cpu_worker, gpu_worker, broadcast_worker, allgather_worker]
)
def test_stateless_process_group(worker):
port1 = get_open_port()
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("", port1))
port2 = get_open_port()
WORLD_SIZE = 4
from multiprocessing import get_context
ctx = get_context("fork")
processes = []
for i in range(WORLD_SIZE):
rank = i
processes.append(
ctx.Process(target=worker, args=(rank, WORLD_SIZE, port1, port2))
)
for p in processes:
p.start()
for p in processes:
p.join()
for p in processes:
assert not p.exitcode
print("All processes finished.")
File diff suppressed because it is too large Load Diff