[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,95 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import atexit
|
||||
import os
|
||||
import random
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
mp.set_start_method("spawn", force=True)
|
||||
|
||||
|
||||
def _distributed_worker_wrapper(fn, env, world_size, args, rank, skip_queue):
|
||||
try:
|
||||
fn(env, world_size, *args)
|
||||
except BaseException as exc:
|
||||
if isinstance(exc, pytest.skip.Exception):
|
||||
skip_queue.put((rank, str(exc)))
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
def distributed_run(fn, world_size, *args):
|
||||
number_of_processes = world_size
|
||||
processes: list[mp.Process] = []
|
||||
skip_queue: mp.SimpleQueue = mp.SimpleQueue()
|
||||
for i in range(number_of_processes):
|
||||
env: dict[str, str] = {}
|
||||
env["RANK"] = str(i)
|
||||
env["LOCAL_RANK"] = str(i)
|
||||
env["WORLD_SIZE"] = str(number_of_processes)
|
||||
env["LOCAL_WORLD_SIZE"] = str(number_of_processes)
|
||||
env["MASTER_ADDR"] = "localhost"
|
||||
env["MASTER_PORT"] = "12345"
|
||||
p = mp.Process(
|
||||
target=_distributed_worker_wrapper,
|
||||
args=(fn, env, world_size, args, i, skip_queue),
|
||||
)
|
||||
processes.append(p)
|
||||
p.start()
|
||||
|
||||
for p in processes:
|
||||
p.join()
|
||||
|
||||
skipped: list[tuple[int, str]] = []
|
||||
while not skip_queue.empty():
|
||||
rank, reason = skip_queue.get()
|
||||
skipped.append((rank, reason))
|
||||
|
||||
if len(skipped) == number_of_processes:
|
||||
reason = skipped[0][1]
|
||||
pytest.skip(reason)
|
||||
if 0 < len(skipped) < number_of_processes:
|
||||
skipped_ranks = sorted(rank for rank, _ in skipped)
|
||||
raise AssertionError(
|
||||
"Distributed test had partial skips; expected either all ranks "
|
||||
f"to skip or none. Skipped ranks: {skipped_ranks}, "
|
||||
f"total ranks: {number_of_processes}"
|
||||
)
|
||||
|
||||
for p in processes:
|
||||
assert p.exitcode == 0
|
||||
|
||||
|
||||
def set_env_vars_and_device(env: dict[str, str]) -> None:
|
||||
update_environment_variables(env)
|
||||
local_rank = os.environ["LOCAL_RANK"]
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
|
||||
backend = "xccl" if current_platform.is_xpu() else "nccl"
|
||||
|
||||
# Create a minimal vllm config for init_distributed_environment
|
||||
vllm_config = VllmConfig()
|
||||
with set_current_vllm_config(vllm_config):
|
||||
init_distributed_environment(backend=backend)
|
||||
atexit.register(_destroy_process_group_if_initialized)
|
||||
# Ensure each worker process has the same random seed
|
||||
random.seed(42)
|
||||
torch.manual_seed(42)
|
||||
|
||||
|
||||
def _destroy_process_group_if_initialized() -> None:
|
||||
if torch.distributed.is_available() and torch.distributed.is_initialized():
|
||||
torch.distributed.destroy_process_group()
|
||||
@@ -1,95 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import sys
|
||||
import types
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._aiter_ops as aiter_ops
|
||||
import vllm.distributed as vllm_distributed
|
||||
import vllm.kernels # noqa: F401
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_rocm_aiter_gemma_rmsnorm_falls_back_after_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
class FakeAiterCA:
|
||||
world_size = 8
|
||||
fully_connected = True
|
||||
|
||||
def custom_fused_ar_rms(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
class FakeAiterAllReduce:
|
||||
aiter_ca = FakeAiterCA()
|
||||
|
||||
def use_1stage_fused_ar_rms(self, inp):
|
||||
return False
|
||||
|
||||
class FakeQuickReduce:
|
||||
disabled = False
|
||||
|
||||
def should_quick_allreduce(self, inp):
|
||||
raise AssertionError("Gemma norm must not use fused QR+RMSNorm")
|
||||
|
||||
device_comm = SimpleNamespace(qr_comm=FakeQuickReduce())
|
||||
monkeypatch.setattr(
|
||||
vllm_distributed,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(device_communicator=device_comm),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
aiter_ops.rocm_aiter_ops,
|
||||
"get_aiter_allreduce",
|
||||
lambda: FakeAiterAllReduce(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
vllm_distributed,
|
||||
"tensor_model_parallel_all_reduce",
|
||||
lambda inp: inp + 2,
|
||||
)
|
||||
|
||||
calls = []
|
||||
fake_aiter = types.SimpleNamespace()
|
||||
|
||||
def fake_rmsnorm2d_fwd_with_add(
|
||||
out,
|
||||
inp,
|
||||
residual,
|
||||
residual_out,
|
||||
weight,
|
||||
epsilon,
|
||||
*,
|
||||
gemma_norm,
|
||||
):
|
||||
calls.append((inp, weight, epsilon, gemma_norm))
|
||||
residual_out.copy_(inp + residual)
|
||||
out.copy_(residual_out)
|
||||
|
||||
fake_aiter.rmsnorm2d_fwd_with_add = fake_rmsnorm2d_fwd_with_add
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
|
||||
# On the platform's device, not the CPU. The fallback leaves Python through
|
||||
# `torch.ops.vllm_aiter.fused_add_rms_norm`, which is registered for the
|
||||
# platform's dispatch key alone, so CPU tensors have nothing to dispatch to
|
||||
# on a GPU build. The kernel behind it is still the fake above.
|
||||
device = current_platform.device_type
|
||||
inp = torch.randn(8192, 16, dtype=torch.bfloat16, device=device)
|
||||
residual = torch.randn_like(inp)
|
||||
weight = torch.randn(16, dtype=torch.bfloat16, device=device)
|
||||
|
||||
out, residual_out = aiter_ops._rocm_aiter_fused_allreduce_rmsnorm_impl(
|
||||
inp, residual, weight, 1e-6, gemma_norm=True
|
||||
)
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0] is not inp
|
||||
assert calls[0][1] is weight
|
||||
assert calls[0][2] == 1e-6
|
||||
assert calls[0][3] is True
|
||||
torch.testing.assert_close(residual_out, inp + 2 + residual)
|
||||
torch.testing.assert_close(out, residual_out)
|
||||
@@ -1,785 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test the communication operators.
|
||||
|
||||
Run `pytest tests/distributed/test_comm_ops.py`.
|
||||
"""
|
||||
|
||||
from collections import deque
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
|
||||
from vllm.distributed import (
|
||||
broadcast_tensor_dict,
|
||||
get_pp_group,
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_all_reduce,
|
||||
tensor_model_parallel_reduce_scatter,
|
||||
)
|
||||
from vllm.distributed.device_communicators import flashinfer_all_reduce
|
||||
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
|
||||
from vllm.distributed.parallel_state import GroupCoordinator, TensorMetadata
|
||||
from vllm.v1.worker.gpu_worker import AsyncIntermediateTensors
|
||||
|
||||
from ..utils import (
|
||||
init_test_distributed_environment,
|
||||
multi_gpu_test,
|
||||
multi_process_parallel,
|
||||
)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def all_reduce_test_worker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
rank: int,
|
||||
distributed_init_port: str,
|
||||
):
|
||||
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
|
||||
# so that each worker can see all the GPUs
|
||||
# they will be able to set the device to the correct GPU
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
num_elements = 8
|
||||
all_tensors = [
|
||||
torch.arange(num_elements, dtype=torch.float32, device="cuda") * (r + 1)
|
||||
for r in range(tp_size)
|
||||
]
|
||||
expected = torch.sum(torch.stack(all_tensors, dim=0), dim=0)
|
||||
t = all_tensors[rank % tp_size]
|
||||
t = tensor_model_parallel_all_reduce(t)
|
||||
torch.testing.assert_close(t, expected)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def reduce_scatter_test_worker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
rank: int,
|
||||
distributed_init_port: str,
|
||||
):
|
||||
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
|
||||
# so that each worker can see all the GPUs
|
||||
# they will be able to set the device to the correct GPU
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
|
||||
num_elements = 8
|
||||
all_tensors = [
|
||||
torch.arange(num_elements, dtype=torch.float32, device="cuda") * (r + 1)
|
||||
for r in range(tp_size)
|
||||
]
|
||||
|
||||
index = rank % tp_size
|
||||
partition_size = num_elements // tp_size
|
||||
all_reduce = torch.sum(torch.stack(all_tensors, dim=0), dim=0)
|
||||
expected = all_reduce[index * partition_size : (index + 1) * partition_size]
|
||||
t = all_tensors[index]
|
||||
t = tensor_model_parallel_reduce_scatter(t, 0)
|
||||
torch.testing.assert_close(t, expected)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def all_gather_test_worker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
rank: int,
|
||||
distributed_init_port: str,
|
||||
):
|
||||
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
|
||||
# so that each worker can see all the GPUs
|
||||
# they will be able to set the device to the correct GPU
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
num_dimensions = 3
|
||||
tensor_size = list(range(2, num_dimensions + 2))
|
||||
total_size = 1
|
||||
for s in tensor_size:
|
||||
total_size *= s
|
||||
for all_gather_dimension in range(num_dimensions):
|
||||
all_tensors = [
|
||||
torch.arange(total_size, dtype=torch.float32, device="cuda").reshape(
|
||||
tensor_size
|
||||
)
|
||||
* (r + 1)
|
||||
for r in range(tp_size)
|
||||
]
|
||||
expected = torch.cat(all_tensors, dim=all_gather_dimension)
|
||||
t = all_tensors[rank % tp_size]
|
||||
t = tensor_model_parallel_all_gather(t, all_gather_dimension)
|
||||
torch.testing.assert_close(t, expected)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def broadcast_tensor_dict_test_worker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
rank: int,
|
||||
distributed_init_port: str,
|
||||
):
|
||||
# it is important to delete the CUDA_VISIBLE_DEVICES environment variable
|
||||
# so that each worker can see all the GPUs
|
||||
# they will be able to set the device to the correct GPU
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
test_dict = {
|
||||
# device tensor
|
||||
"a": torch.arange(8, dtype=torch.float32, device="cuda"),
|
||||
# CPU tensor
|
||||
"b": torch.arange(16, dtype=torch.int8, device="cpu"),
|
||||
"c": "test",
|
||||
"d": [1, 2, 3],
|
||||
"e": {"a": 1, "b": 2},
|
||||
# empty tensor
|
||||
"f": torch.tensor([], dtype=torch.float32, device="cuda"),
|
||||
}
|
||||
|
||||
if (rank % tp_size) == 0:
|
||||
broadcast_tensor_dict(test_dict, src=0)
|
||||
else:
|
||||
recv_dict = broadcast_tensor_dict(src=0)
|
||||
assert len(recv_dict) == len(test_dict)
|
||||
torch.testing.assert_close(recv_dict["a"], test_dict["a"])
|
||||
torch.testing.assert_close(recv_dict["b"], test_dict["b"])
|
||||
assert recv_dict["c"] == test_dict["c"]
|
||||
assert recv_dict["d"] == test_dict["d"]
|
||||
assert recv_dict["e"] == test_dict["e"]
|
||||
torch.testing.assert_close(recv_dict["f"], test_dict["f"])
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def send_recv_tensor_dict_test_worker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
rank: int,
|
||||
distributed_init_port: str,
|
||||
):
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
|
||||
test_dict = {
|
||||
# device tensor
|
||||
"a": torch.arange(8, dtype=torch.float32, device="cuda"),
|
||||
# CPU tensor
|
||||
"b": torch.arange(16, dtype=torch.int8, device="cpu"),
|
||||
"c": "test",
|
||||
"d": [1, 2, 3],
|
||||
"e": {"a": 1, "b": 2},
|
||||
# empty tensor
|
||||
"f": torch.tensor([], dtype=torch.float32, device="cuda"),
|
||||
}
|
||||
|
||||
if not get_pp_group().is_first_rank:
|
||||
recv_dict = get_pp_group().recv_tensor_dict()
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
get_pp_group().send_tensor_dict(test_dict)
|
||||
|
||||
if not get_pp_group().is_first_rank:
|
||||
assert len(recv_dict) == len(test_dict)
|
||||
torch.testing.assert_close(recv_dict["a"], test_dict["a"])
|
||||
torch.testing.assert_close(recv_dict["b"], test_dict["b"])
|
||||
assert recv_dict["c"] == test_dict["c"]
|
||||
assert recv_dict["d"] == test_dict["d"]
|
||||
assert recv_dict["e"] == test_dict["e"]
|
||||
torch.testing.assert_close(recv_dict["f"], test_dict["f"])
|
||||
|
||||
|
||||
class _DummyWork:
|
||||
def __init__(self) -> None:
|
||||
self.wait_calls = 0
|
||||
self.completed = False
|
||||
|
||||
def wait(self) -> None:
|
||||
self.wait_calls += 1
|
||||
self.completed = True
|
||||
|
||||
def is_completed(self) -> bool:
|
||||
return self.completed
|
||||
|
||||
|
||||
class _DummyAllGatherGroup:
|
||||
def __init__(self, world_size: int, rank_in_group: int) -> None:
|
||||
self.world_size = world_size
|
||||
self.rank_in_group = rank_in_group
|
||||
|
||||
def all_gather(self, t: torch.Tensor, dim: int = 0) -> torch.Tensor:
|
||||
# duplicate local slice across ranks.
|
||||
assert dim == 0
|
||||
return torch.cat([t for _ in range(self.world_size)], dim=0)
|
||||
|
||||
|
||||
def _make_group_for_unit_test(
|
||||
rank_in_group: int = 0, world_size: int = 2
|
||||
) -> GroupCoordinator:
|
||||
# avoid running GroupCoordinator.__init__ (it wires up real process groups).
|
||||
g = GroupCoordinator.__new__(GroupCoordinator)
|
||||
g.world_size = world_size
|
||||
g.rank_in_group = rank_in_group
|
||||
g.ranks = list(range(world_size))
|
||||
g.use_cpu_custom_send_recv = False
|
||||
g.device_group = None
|
||||
g.cpu_group = None
|
||||
g._pending_isends = deque()
|
||||
return g
|
||||
|
||||
|
||||
def test_irecv_tensor_dict_send_allgather_postprocess_binds_keys(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
def fake_irecv(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
|
||||
t.fill_(1)
|
||||
return _DummyWork()
|
||||
|
||||
monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True)
|
||||
monkeypatch.setattr(torch.distributed, "irecv", fake_irecv)
|
||||
|
||||
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
|
||||
# 2 tensors so we can catch late-binding bugs in postprocess closures.
|
||||
metadata_list = [
|
||||
("a", TensorMetadata("cpu", torch.int32, torch.Size([4]))),
|
||||
("b", TensorMetadata("cpu", torch.int32, torch.Size([4]))),
|
||||
]
|
||||
g.recv_object = lambda src=None: metadata_list # type: ignore[method-assign]
|
||||
|
||||
ag = _DummyAllGatherGroup(world_size=2, rank_in_group=0)
|
||||
td, handles, postprocess = g.irecv_tensor_dict(all_gather_group=ag)
|
||||
|
||||
assert td is not None
|
||||
assert len(handles) == 2
|
||||
assert len(postprocess) == 2
|
||||
|
||||
# before postprocess, dict holds the TP slice (shape 2).
|
||||
assert td["a"].shape == torch.Size([2])
|
||||
assert td["b"].shape == torch.Size([2])
|
||||
|
||||
# simulate worker-side "defer wait": wait + postprocess later.
|
||||
for handle in handles:
|
||||
handle.wait()
|
||||
for fn in postprocess:
|
||||
fn()
|
||||
|
||||
# after postprocess, dict values are reconstructed to full shape (shape 4),
|
||||
# and each key should be updated independently
|
||||
assert td["a"].shape == torch.Size([4])
|
||||
assert td["b"].shape == torch.Size([4])
|
||||
torch.testing.assert_close(td["a"], torch.ones(4, dtype=torch.int32))
|
||||
torch.testing.assert_close(td["b"], torch.ones(4, dtype=torch.int32))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("aliased", [False, True])
|
||||
def test_cuda_communicator_checkpoints_flashinfer_workspaces(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
aliased: bool,
|
||||
) -> None:
|
||||
group = object()
|
||||
normal_workspace = Mock()
|
||||
quant_workspace = normal_workspace if aliased else Mock()
|
||||
unique_workspaces = (
|
||||
[normal_workspace] if aliased else [normal_workspace, quant_workspace]
|
||||
)
|
||||
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", normal_workspace)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce, "_fi_ar_quant_workspace", quant_workspace
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce,
|
||||
"_fi_ar_workspace_groups",
|
||||
{id(workspace): group for workspace in unique_workspaces},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce, "TorchDistBackend", lambda group: group, raising=False
|
||||
)
|
||||
|
||||
communicator = CudaCommunicator.__new__(CudaCommunicator)
|
||||
communicator.cpu_group = group
|
||||
communicator.fi_ar_comm = None
|
||||
communicator.all2all_manager = None
|
||||
communicator.checkpoint_prepare()
|
||||
communicator.checkpoint_restore()
|
||||
|
||||
for workspace in unique_workspaces:
|
||||
workspace.checkpoint_prepare.assert_called_once_with()
|
||||
workspace.checkpoint_restore.assert_called_once_with(group)
|
||||
|
||||
|
||||
def _fi_ar_communicator(
|
||||
monkeypatch: pytest.MonkeyPatch, group: object, normal: Mock, quant: Mock
|
||||
) -> CudaCommunicator:
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", normal)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", quant)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce,
|
||||
"_fi_ar_workspace_groups",
|
||||
{id(normal): group, id(quant): group},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce, "TorchDistBackend", lambda group: group, raising=False
|
||||
)
|
||||
communicator = CudaCommunicator.__new__(CudaCommunicator)
|
||||
communicator.cpu_group = group
|
||||
communicator.pynccl_comm = Mock()
|
||||
communicator.all2all_manager = None
|
||||
return communicator
|
||||
|
||||
|
||||
def _symm_mem_workspace() -> Mock:
|
||||
workspace = Mock()
|
||||
workspace.checkpoint_prepare.side_effect = NotImplementedError
|
||||
workspace.checkpoint_restore.side_effect = NotImplementedError
|
||||
return workspace
|
||||
|
||||
|
||||
@pytest.mark.parametrize("symm_mem", [False, True], ids=["mnnvl", "symm-mem+mnnvl"])
|
||||
def test_cuda_communicator_suspend_detaches_flashinfer_workspaces(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
symm_mem: bool,
|
||||
) -> None:
|
||||
group = object()
|
||||
workspaces = (_symm_mem_workspace() if symm_mem else Mock(), Mock())
|
||||
communicator = _fi_ar_communicator(monkeypatch, group, *workspaces)
|
||||
communicator.suspend()
|
||||
communicator.resume()
|
||||
|
||||
for workspace in workspaces:
|
||||
workspace.checkpoint_prepare.assert_called_once_with()
|
||||
workspace.checkpoint_restore.assert_called_once_with(group)
|
||||
communicator.pynccl_comm.suspend.assert_called_once_with()
|
||||
communicator.pynccl_comm.resume.assert_called_once_with()
|
||||
|
||||
|
||||
def test_cuda_communicator_checkpoint_rejects_symm_mem_workspace(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
communicator = _fi_ar_communicator(
|
||||
monkeypatch, object(), _symm_mem_workspace(), Mock()
|
||||
)
|
||||
with pytest.raises(NotImplementedError):
|
||||
communicator.checkpoint_prepare()
|
||||
with pytest.raises(NotImplementedError):
|
||||
communicator.checkpoint_restore()
|
||||
|
||||
|
||||
def test_cuda_communicator_resume_retries_failed_restore(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
workspace = Mock()
|
||||
workspace.checkpoint_restore.side_effect = [RuntimeError("cuMemCreate"), None]
|
||||
communicator = _fi_ar_communicator(monkeypatch, object(), workspace, Mock())
|
||||
communicator.suspend()
|
||||
with pytest.raises(RuntimeError):
|
||||
communicator.resume()
|
||||
communicator.resume()
|
||||
|
||||
assert workspace.checkpoint_restore.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "capability", "world_size", "nodes", "expected"),
|
||||
[
|
||||
("mnnvl", 103, 4, 1, 80 * flashinfer_all_reduce.MiB - 1),
|
||||
("mnnvl", 103, 8, 2, 64 * flashinfer_all_reduce.MiB - 1),
|
||||
("mnnvl", 103, 16, 4, 8 * flashinfer_all_reduce.MiB - 1),
|
||||
("mnnvl", 103, 2, 1, None),
|
||||
("mnnvl", 103, 12, 3, None),
|
||||
("mnnvl", 103, 8, 1, 64 * flashinfer_all_reduce.MiB - 1),
|
||||
("trtllm", 103, 8, 2, None),
|
||||
("mnnvl", 90, 8, 2, None),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_standalone_size_tuning(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
backend: str,
|
||||
capability: int,
|
||||
world_size: int,
|
||||
nodes: int,
|
||||
expected: int | None,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce,
|
||||
"current_platform",
|
||||
Mock(get_device_capability=lambda: Mock(to_int=lambda: capability)),
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_node_count", lambda _: nodes)
|
||||
|
||||
assert (
|
||||
flashinfer_all_reduce._get_tuned_standalone_max_size(
|
||||
world_size, backend, Mock()
|
||||
)
|
||||
== expected
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("enabled", "expected"), [(True, 4681), (False, 128)])
|
||||
def test_flashinfer_standalone_workspace_size(
|
||||
monkeypatch: pytest.MonkeyPatch, enabled: bool, expected: int
|
||||
) -> None:
|
||||
create_workspace = Mock(return_value=Mock(backend="mnnvl"))
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce.envs, "VLLM_ALLREDUCE_USE_FLASHINFER", enabled
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", None)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", None)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce,
|
||||
"_resolve_fi_ar_backend",
|
||||
Mock(return_value=("mnnvl", False)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce,
|
||||
"_get_tuned_standalone_max_size",
|
||||
Mock(return_value=64 * flashinfer_all_reduce.MiB - 1),
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_create_workspace", create_workspace)
|
||||
|
||||
flashinfer_all_reduce.get_fi_ar_workspace(8, 0, 128, 7168, torch.bfloat16, Mock())
|
||||
|
||||
assert create_workspace.call_args.args[3] == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("node_count", "is_blackwell", "expected"),
|
||||
[
|
||||
(1, False, ("mnnvl", True)),
|
||||
(1, True, ("mnnvl", True)),
|
||||
(2, False, (None, False)),
|
||||
(2, True, ("mnnvl", False)),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_auto_backend_uses_group_topology(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
node_count: int,
|
||||
is_blackwell: bool,
|
||||
expected: tuple[str | None, bool],
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce.envs, "VLLM_FLASHINFER_ALLREDUCE_BACKEND", "auto"
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_node_count", lambda _: node_count)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce.current_platform,
|
||||
"has_device_capability",
|
||||
lambda capability: is_blackwell,
|
||||
)
|
||||
|
||||
assert flashinfer_all_reduce._resolve_fi_ar_backend(Mock()) == expected
|
||||
|
||||
|
||||
def test_flashinfer_workspace_failure_is_not_retried(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Creation is collective; retrying it per all-reduce desyncs ranks."""
|
||||
create_workspace = Mock(return_value=None)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", None)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", None)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace_failed", False)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce,
|
||||
"_resolve_fi_ar_backend",
|
||||
Mock(return_value=("mnnvl", True)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None)
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_create_workspace", create_workspace)
|
||||
args = (2, 0, 128, 4096, torch.bfloat16, Mock())
|
||||
|
||||
assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None
|
||||
assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None
|
||||
# One mnnvl attempt plus one trtllm fallback, never repeated.
|
||||
assert create_workspace.call_count == 2
|
||||
|
||||
flashinfer_all_reduce.destroy_fi_ar_workspace()
|
||||
flashinfer_all_reduce.get_fi_ar_workspace(*args)
|
||||
assert create_workspace.call_count == 4
|
||||
|
||||
|
||||
def test_flashinfer_all_reduce_precedes_nccl(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
output = torch.empty(2)
|
||||
fi_ar_comm = Mock(disabled=False)
|
||||
fi_ar_comm.should_use_fi_ar.return_value = True
|
||||
fi_ar_comm.all_reduce.return_value = output
|
||||
communicator = CudaCommunicator.__new__(CudaCommunicator)
|
||||
communicator.fi_ar_comm = fi_ar_comm
|
||||
communicator.fi_pcie_ipc_ar_comm = None
|
||||
communicator.pynccl_comm = Mock(world_size=8)
|
||||
communicator.qr_comm = None
|
||||
nccl_selector = Mock(return_value=True)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.device_communicators.cuda_communicator."
|
||||
"should_nccl_symm_mem_allreduce",
|
||||
nccl_selector,
|
||||
)
|
||||
|
||||
assert communicator.all_reduce(torch.empty(1)) is output
|
||||
nccl_selector.assert_not_called()
|
||||
|
||||
|
||||
def test_aiter_all_gather_precedes_pynccl(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Ensure can use aiter all_gather if available even if pynccl is not available."""
|
||||
output = torch.empty(2)
|
||||
aiter_comm = Mock()
|
||||
aiter_comm.should_custom_ag.return_value = True
|
||||
aiter_comm.custom_all_gather.return_value = output
|
||||
communicator = CudaCommunicator.__new__(CudaCommunicator)
|
||||
communicator.world_size = 2
|
||||
communicator.aiter_ar_comm = aiter_comm
|
||||
communicator.pynccl_comm = Mock(disabled=True)
|
||||
monkeypatch.setattr(communicator, "_can_use_aiter_ag_rs", Mock(return_value=True))
|
||||
|
||||
assert communicator.all_gatherv(torch.empty(1)) is output
|
||||
|
||||
|
||||
def test_isend_object_posts_size_then_object_and_releases_on_wait(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
posted: list[tuple[torch.Tensor, _DummyWork]] = []
|
||||
|
||||
def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
|
||||
w = _DummyWork()
|
||||
posted.append((t, w))
|
||||
return w
|
||||
|
||||
monkeypatch.setattr(torch.distributed, "isend", fake_isend)
|
||||
|
||||
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
|
||||
handle = g.isend_object({"k": [1, 2, 3]}, dst=1)
|
||||
|
||||
# two sends, in size-then-object order (preserves gloo FIFO).
|
||||
assert len(posted) == 2
|
||||
assert posted[0][0].dtype == torch.long
|
||||
assert posted[0][0].shape == torch.Size([1])
|
||||
assert posted[0][0].item() == posted[1][0].numel()
|
||||
assert posted[1][0].dtype == torch.uint8
|
||||
|
||||
# retain holds both source tensors until wait.
|
||||
assert handle._retained == (posted[0][0], posted[1][0])
|
||||
|
||||
handle.wait()
|
||||
|
||||
# both underlying works drained, retain dropped.
|
||||
assert all(w.wait_calls == 1 for _, w in posted)
|
||||
assert handle._retained == ()
|
||||
|
||||
# wait is idempotent: gloo Work.wait() is single-shot for p2p sends (a
|
||||
# second wait blocks forever), and both the lazy FIFO reap in
|
||||
# ``_reap_completed_isends`` and any explicit isend caller may wait the
|
||||
# same metadata handle.
|
||||
handle.wait()
|
||||
assert all(w.wait_calls == 1 for _, w in posted)
|
||||
assert handle._retained == ()
|
||||
|
||||
|
||||
def test_isend_tensor_dict_includes_metadata_handle(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
posted: list[torch.Tensor] = []
|
||||
|
||||
def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
|
||||
posted.append(t)
|
||||
return _DummyWork()
|
||||
|
||||
monkeypatch.setattr(torch.distributed, "isend", fake_isend)
|
||||
|
||||
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
|
||||
td = {"a": torch.arange(4, dtype=torch.float32, device="cpu")}
|
||||
handles = g.isend_tensor_dict(td, dst=1)
|
||||
|
||||
# size + object (metadata) + one tensor send.
|
||||
assert len(posted) == 3
|
||||
assert posted[0].dtype == torch.long
|
||||
assert posted[1].dtype == torch.uint8
|
||||
assert posted[2].dtype == torch.float32
|
||||
|
||||
# composite metadata handle + one tensor handle.
|
||||
assert len(handles) == 2
|
||||
|
||||
for handle in handles:
|
||||
handle.wait()
|
||||
assert handles[0]._retained == ()
|
||||
|
||||
|
||||
def test_isend_tensor_dict_self_retains_for_fire_and_forget(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
works: list[_DummyWork] = []
|
||||
|
||||
def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
|
||||
w = _DummyWork()
|
||||
works.append(w)
|
||||
return w
|
||||
|
||||
monkeypatch.setattr(torch.distributed, "isend", fake_isend)
|
||||
|
||||
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
|
||||
td = {"a": torch.arange(4, dtype=torch.float32, device="cpu")}
|
||||
|
||||
# fire-and-forget: the returned handles are dropped by the caller, so the
|
||||
# group must self-retain the handles and the source tensor.
|
||||
g.isend_tensor_dict(td, dst=1)
|
||||
assert len(g._pending_isends) == 1
|
||||
handles0, tensors0 = g._pending_isends[0]
|
||||
assert len(handles0) == 2 # metadata composite + one tensor send
|
||||
assert tensors0 == [td["a"]]
|
||||
|
||||
# the first send is still in flight, so the next call's reap keeps it.
|
||||
g.isend_tensor_dict(td, dst=1)
|
||||
assert len(g._pending_isends) == 2
|
||||
|
||||
# metadata-only send (no tensors) is dropped best-effort on the next
|
||||
# reap, since it has no reliable completion signal.
|
||||
g.isend_tensor_dict({"meta": "no-tensors"}, dst=1)
|
||||
assert len(g._pending_isends) == 3
|
||||
|
||||
# complete every posted work; the next call reaps all three old entries
|
||||
# (the metadata-only one unconditionally) and keeps only its own.
|
||||
for w in works:
|
||||
w.completed = True
|
||||
g.isend_tensor_dict(td, dst=1)
|
||||
assert len(g._pending_isends) == 1
|
||||
# reaped entries had their metadata handles waited as a backstop.
|
||||
assert handles0[0]._retained == ()
|
||||
|
||||
|
||||
def test_send_tensor_dict_sync_path_does_not_self_retain(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
|
||||
return _DummyWork()
|
||||
|
||||
monkeypatch.setattr(torch.distributed, "isend", fake_isend)
|
||||
monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True)
|
||||
monkeypatch.setattr(GroupCoordinator, "send_object", lambda self, obj, dst: None)
|
||||
|
||||
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
|
||||
td = {"a": torch.arange(4, dtype=torch.float32, device="cpu")}
|
||||
g.send_tensor_dict(td, dst=1)
|
||||
|
||||
# the sync path waited every handle, so no retention entry may leak.
|
||||
assert len(g._pending_isends) == 0
|
||||
|
||||
|
||||
def test_async_intermediate_tensors_lazy_wait() -> None:
|
||||
work = _DummyWork()
|
||||
post_calls = {"n": 0}
|
||||
|
||||
def post() -> None:
|
||||
post_calls["n"] += 1
|
||||
|
||||
it = AsyncIntermediateTensors(
|
||||
{"x": torch.tensor([1])},
|
||||
comm_handles=[work],
|
||||
comm_postprocess=[post],
|
||||
)
|
||||
|
||||
# accessing non-tensor attributes should not trigger wait.
|
||||
assert it._comm_handles is not None
|
||||
assert work.wait_calls == 0
|
||||
assert post_calls["n"] == 0
|
||||
|
||||
# first access of `.tensors` triggers wait + postprocess.
|
||||
_ = it.tensors
|
||||
assert work.wait_calls == 1
|
||||
assert post_calls["n"] == 1
|
||||
|
||||
# subsequent access should not re-wait.
|
||||
_ = it.tensors
|
||||
assert work.wait_calls == 1
|
||||
assert post_calls["n"] == 1
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def send_recv_test_worker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
rank: int,
|
||||
distributed_init_port: str,
|
||||
):
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
|
||||
size = 64
|
||||
test_tensor = torch.arange(64, dtype=torch.float32, device="cuda")
|
||||
|
||||
if not get_pp_group().is_first_rank:
|
||||
recv_tensor = get_pp_group().recv(size, dtype=torch.float32)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
get_pp_group().send(test_tensor)
|
||||
|
||||
if not get_pp_group().is_first_rank:
|
||||
torch.testing.assert_close(test_tensor, recv_tensor)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("tp_size", [2])
|
||||
@pytest.mark.parametrize(
|
||||
"test_target",
|
||||
[all_reduce_test_worker, all_gather_test_worker, broadcast_tensor_dict_test_worker],
|
||||
)
|
||||
def test_multi_process_tensor_parallel(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
test_target: Callable[..., Any],
|
||||
):
|
||||
multi_process_parallel(monkeypatch, tp_size, 1, test_target)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("pp_size", [2])
|
||||
@pytest.mark.parametrize(
|
||||
"test_target", [send_recv_test_worker, send_recv_tensor_dict_test_worker]
|
||||
)
|
||||
def test_multi_process_pipeline_parallel(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
pp_size: int,
|
||||
test_target: Callable[..., Any],
|
||||
):
|
||||
multi_process_parallel(monkeypatch, 1, pp_size, test_target)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
@pytest.mark.parametrize("tp_size", [2])
|
||||
@pytest.mark.parametrize("pp_size", [2])
|
||||
@pytest.mark.parametrize(
|
||||
"test_target",
|
||||
[
|
||||
send_recv_test_worker,
|
||||
send_recv_tensor_dict_test_worker,
|
||||
all_reduce_test_worker,
|
||||
all_gather_test_worker,
|
||||
broadcast_tensor_dict_test_worker,
|
||||
],
|
||||
)
|
||||
def test_multi_process_tensor_parallel_pipeline_parallel(
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
test_target: Callable[..., Any],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
multi_process_parallel(monkeypatch, tp_size, pp_size, test_target)
|
||||
@@ -1,318 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""WARNING: This test runs in both single-node (4 GPUs) and multi-node
|
||||
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
|
||||
important to set the distributed backend to "mp" to avoid Ray scheduling
|
||||
all workers in a node other than the head node, which can cause the test
|
||||
to fail.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal, NamedTuple
|
||||
|
||||
import lm_eval
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import RemoteOpenAIServer, create_new_process_for_each_test
|
||||
from vllm.config.model import RunnerOption
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..models.registry import HF_EXAMPLE_MODELS
|
||||
|
||||
logger = init_logger("test_context_parallel")
|
||||
|
||||
VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
|
||||
|
||||
CP_TEST_MODELS = [
|
||||
# TODO support other models
|
||||
# [LANGUAGE GENERATION]
|
||||
"deepseek-ai/DeepSeek-V2-Lite-Chat",
|
||||
"Qwen/Qwen2.5-1.5B-Instruct",
|
||||
"Qwen/Qwen3.5-0.8B", # hybrid attention model
|
||||
]
|
||||
|
||||
# GSM8K eval configuration
|
||||
NUM_SHOTS = 5 # Few-shot examples
|
||||
TASK = "gsm8k"
|
||||
FILTER = "exact_match,strict-match"
|
||||
NUM_CONCURRENT = 128
|
||||
# tp accuracy with 2% buffer
|
||||
MIN_ACCURACY = {
|
||||
# .buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml
|
||||
"deepseek-ai/DeepSeek-V2-Lite-Chat": 0.64,
|
||||
# .buildkite/lm-eval-harness/configs/Qwen2.5-1.5B-Instruct.yaml
|
||||
"Qwen/Qwen2.5-1.5B-Instruct": 0.52,
|
||||
"Qwen/Qwen3.5-0.8B": 0.33,
|
||||
}
|
||||
|
||||
|
||||
class ParallelSetup(NamedTuple):
|
||||
tp_size: int
|
||||
pp_size: int
|
||||
dcp_size: int
|
||||
cp_kv_cache_interleave_size: int
|
||||
eager_mode: bool
|
||||
chunked_prefill: bool
|
||||
|
||||
|
||||
class CPTestOptions(NamedTuple):
|
||||
multi_node_only: bool
|
||||
attn_backend: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CPTestSettings:
|
||||
parallel_setups: list[ParallelSetup]
|
||||
distributed_backends: list[str]
|
||||
runner: RunnerOption
|
||||
test_options: CPTestOptions
|
||||
|
||||
@staticmethod
|
||||
def detailed(
|
||||
*,
|
||||
tp_base: int = 4,
|
||||
pp_base: int = 1,
|
||||
dcp_multipliers: list[float] | None = None,
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
multi_node_only: bool = False,
|
||||
runner: RunnerOption = "auto",
|
||||
attn_backend: str | None = None,
|
||||
):
|
||||
parallel_setups = []
|
||||
if dcp_multipliers is None:
|
||||
dcp_multipliers = [
|
||||
0.5,
|
||||
]
|
||||
for eager_mode_val in [False]:
|
||||
for pp_multiplier in [1]:
|
||||
for dcp_multiplier in dcp_multipliers:
|
||||
for chunked_prefill_val in [True]:
|
||||
parallel_setups.append(
|
||||
ParallelSetup(
|
||||
tp_size=tp_base,
|
||||
pp_size=pp_multiplier * pp_base,
|
||||
dcp_size=int(dcp_multiplier * tp_base),
|
||||
cp_kv_cache_interleave_size=cp_kv_cache_interleave_size,
|
||||
eager_mode=eager_mode_val,
|
||||
chunked_prefill=chunked_prefill_val,
|
||||
)
|
||||
)
|
||||
return CPTestSettings(
|
||||
parallel_setups=parallel_setups,
|
||||
distributed_backends=["mp"],
|
||||
runner=runner,
|
||||
test_options=CPTestOptions(
|
||||
multi_node_only=multi_node_only,
|
||||
attn_backend=attn_backend,
|
||||
),
|
||||
)
|
||||
|
||||
def iter_params(self, model_id: str):
|
||||
opts = self.test_options
|
||||
|
||||
for parallel_setup in self.parallel_setups:
|
||||
for backend in self.distributed_backends:
|
||||
yield (
|
||||
model_id,
|
||||
parallel_setup,
|
||||
backend,
|
||||
self.runner,
|
||||
opts,
|
||||
)
|
||||
|
||||
|
||||
if current_platform.is_rocm():
|
||||
CP_TEXT_GENERATION_MODELS = {
|
||||
"deepseek-ai/DeepSeek-V2-Lite-Chat": [
|
||||
CPTestSettings.detailed(dcp_multipliers=[1]),
|
||||
],
|
||||
"Qwen/Qwen2.5-1.5B-Instruct": [
|
||||
CPTestSettings.detailed(dcp_multipliers=[1]),
|
||||
],
|
||||
}
|
||||
else:
|
||||
CP_TEXT_GENERATION_MODELS = {
|
||||
"deepseek-ai/DeepSeek-V2-Lite-Chat": [
|
||||
CPTestSettings.detailed(dcp_multipliers=[1]),
|
||||
CPTestSettings.detailed(
|
||||
dcp_multipliers=[0.5],
|
||||
cp_kv_cache_interleave_size=64,
|
||||
attn_backend="FLASHMLA",
|
||||
),
|
||||
],
|
||||
"Qwen/Qwen2.5-1.5B-Instruct": [
|
||||
CPTestSettings.detailed(
|
||||
cp_kv_cache_interleave_size=16, attn_backend="FLASH_ATTN"
|
||||
),
|
||||
CPTestSettings.detailed(
|
||||
cp_kv_cache_interleave_size=16, attn_backend="FLASHINFER"
|
||||
),
|
||||
],
|
||||
"Qwen/Qwen3.5-0.8B": [
|
||||
CPTestSettings.detailed(
|
||||
cp_kv_cache_interleave_size=16,
|
||||
attn_backend="FLASH_ATTN",
|
||||
),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _test_cp_gsm8k(
|
||||
model_id: str,
|
||||
parallel_setup: ParallelSetup,
|
||||
distributed_backend: str,
|
||||
runner: RunnerOption,
|
||||
test_options: CPTestOptions,
|
||||
num_gpus_available: int,
|
||||
*,
|
||||
method: Literal["generate"],
|
||||
is_multimodal: bool,
|
||||
):
|
||||
(
|
||||
tp_size,
|
||||
pp_size,
|
||||
dcp_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
eager_mode,
|
||||
chunked_prefill,
|
||||
) = parallel_setup
|
||||
|
||||
multi_node_only, attn_backend = test_options
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
trust_remote_code = model_info.trust_remote_code
|
||||
tokenizer_mode = model_info.tokenizer_mode
|
||||
hf_overrides = model_info.hf_overrides
|
||||
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
|
||||
if num_gpus_available < tp_size * pp_size:
|
||||
pytest.skip(f"Need at least {tp_size} x {pp_size} GPUs")
|
||||
if VLLM_MULTI_NODE and distributed_backend == "mp":
|
||||
pytest.skip(
|
||||
"Skipping multi-node pipeline parallel test for "
|
||||
"multiprocessing distributed backend"
|
||||
)
|
||||
if multi_node_only and not VLLM_MULTI_NODE:
|
||||
pytest.skip("Not in multi-node setting")
|
||||
|
||||
server_args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--max-num-seqs",
|
||||
"64",
|
||||
]
|
||||
if chunked_prefill:
|
||||
server_args.append("--enable-chunked-prefill")
|
||||
if eager_mode:
|
||||
server_args.append("--enforce-eager")
|
||||
if runner != "auto":
|
||||
server_args.extend(["--runner", runner])
|
||||
if trust_remote_code:
|
||||
server_args.append("--trust-remote-code")
|
||||
if tokenizer_mode:
|
||||
server_args.extend(["--tokenizer-mode", tokenizer_mode])
|
||||
if hf_overrides:
|
||||
server_args.extend(["--hf-overrides", json.dumps(hf_overrides)])
|
||||
|
||||
server_args.extend(
|
||||
[
|
||||
"--tensor-parallel-size",
|
||||
str(tp_size),
|
||||
"--pipeline-parallel-size",
|
||||
str(pp_size),
|
||||
"--decode-context-parallel-size",
|
||||
str(dcp_size),
|
||||
"--dcp-kv-cache-interleave-size",
|
||||
str(cp_kv_cache_interleave_size),
|
||||
"--distributed-executor-backend",
|
||||
distributed_backend,
|
||||
]
|
||||
)
|
||||
|
||||
if attn_backend:
|
||||
server_args.append(f"--attention-backend={attn_backend}")
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_id,
|
||||
server_args,
|
||||
max_wait_seconds=720,
|
||||
) as remote_server:
|
||||
url = f"{remote_server.url_for('v1')}/completions"
|
||||
|
||||
model_args = (
|
||||
f"model={model_id},"
|
||||
f"base_url={url},"
|
||||
f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False"
|
||||
)
|
||||
|
||||
results = lm_eval.simple_evaluate(
|
||||
model="local-completions",
|
||||
model_args=model_args,
|
||||
tasks=TASK,
|
||||
num_fewshot=NUM_SHOTS,
|
||||
)
|
||||
|
||||
# Validate accuracy is reasonable
|
||||
accuracy = results["results"][TASK][FILTER]
|
||||
min_accuracy = MIN_ACCURACY[model_id]
|
||||
assert accuracy >= min_accuracy, (
|
||||
f"TP+DCP accuracy too low: {accuracy:.3f} < {min_accuracy:.3f}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"model_id",
|
||||
"parallel_setup",
|
||||
"distributed_backend",
|
||||
"runner",
|
||||
"test_options",
|
||||
),
|
||||
[
|
||||
params
|
||||
for model_id, settings in CP_TEXT_GENERATION_MODELS.items()
|
||||
for setting in settings
|
||||
for params in setting.iter_params(model_id)
|
||||
if model_id in CP_TEST_MODELS
|
||||
],
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
def test_cp_generation(
|
||||
model_id: str,
|
||||
parallel_setup: ParallelSetup,
|
||||
distributed_backend: str,
|
||||
runner: RunnerOption,
|
||||
test_options: CPTestOptions,
|
||||
num_gpus_available,
|
||||
):
|
||||
if (
|
||||
model_id == "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
||||
and torch.cuda.get_device_capability() < (9, 0)
|
||||
):
|
||||
pytest.skip(reason="MLA+DCP requires compute capability of 9.0 or higher")
|
||||
if (
|
||||
model_id == "Qwen/Qwen2.5-1.5B-Instruct"
|
||||
and torch.cuda.get_device_capability() != (9, 0)
|
||||
):
|
||||
pytest.skip(reason="GQA+DCP currently requires compute capability of 9.0")
|
||||
|
||||
_test_cp_gsm8k(
|
||||
model_id,
|
||||
parallel_setup,
|
||||
distributed_backend,
|
||||
runner,
|
||||
test_options,
|
||||
num_gpus_available,
|
||||
method="generate",
|
||||
is_multimodal=False,
|
||||
)
|
||||
@@ -1,394 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Validate production Lamport cleanup from a reachable one-epoch skew.
|
||||
|
||||
The fast rank publishes epoch N+1 before the slow rank runs epoch N cleanup.
|
||||
Peer slots are prefilled only so each shipping kernel can finish independently;
|
||||
the test then checks that slow-rank cleanup retained the real N+1 publication.
|
||||
The production kernels are also seeded at the final two uint32 values to verify
|
||||
that stage selection remains bounded and never reuses a stage across wrap.
|
||||
|
||||
Each rank's local buffer contains three all-gather stages followed by three
|
||||
reduce-scatter stages::
|
||||
|
||||
all-gather: [stage 0][stage 1][stage 2]
|
||||
reduce-scatter: [stage 0][stage 1][stage 2]
|
||||
|
||||
Each stage contains one payload slot per source rank.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_test_distributed_environment,
|
||||
multi_process_parallel,
|
||||
)
|
||||
|
||||
_SENTINEL = -2147483648
|
||||
_ROWS = 32
|
||||
_HIDDEN = 8
|
||||
_NUM_STAGES = 3
|
||||
_NUM_RANKS = 2
|
||||
_FAST_RANK = 0
|
||||
_SLOW_RANK = 1
|
||||
_PACK_BYTES = 16
|
||||
_UINT32_MODULUS = 1 << 32
|
||||
_UINT32_MAX = _UINT32_MODULUS - 1
|
||||
_ALL_GATHER_EPOCH_ROW = 0
|
||||
_REDUCE_SCATTER_EPOCH_ROW = 1
|
||||
_REDUCE_SCATTER_REGION_STAGE_OFFSET = 3
|
||||
|
||||
|
||||
def _supports_multimem():
|
||||
capability = current_platform.get_device_capability()
|
||||
return (
|
||||
current_platform.is_cuda() and capability is not None and capability.major >= 9
|
||||
)
|
||||
|
||||
|
||||
def _supports_sm100_or_sm103():
|
||||
capability = current_platform.get_device_capability()
|
||||
return current_platform.is_cuda() and capability in ((10, 0), (10, 3))
|
||||
|
||||
|
||||
def _payload(value, device):
|
||||
return torch.full((_ROWS, _HIDDEN), value, dtype=torch.bfloat16, device=device)
|
||||
|
||||
|
||||
def _payload_bytes(value):
|
||||
return value.contiguous().view(torch.uint8).flatten()
|
||||
|
||||
|
||||
def _int32_bits(value):
|
||||
"""Represent an unsigned 32-bit value in the signed epoch tensor."""
|
||||
value %= _UINT32_MODULUS
|
||||
return value if value < (1 << 31) else value - _UINT32_MODULUS
|
||||
|
||||
|
||||
def _epoch_value(epoch):
|
||||
return int(epoch.item()) % _UINT32_MODULUS
|
||||
|
||||
|
||||
def _stage_slot(buffer, stage_size, region_offset, stage, src_rank, slot_size):
|
||||
offset = region_offset + stage * stage_size + src_rank * slot_size
|
||||
return buffer.narrow(0, offset, slot_size)
|
||||
|
||||
|
||||
def _prepare_rank_state(
|
||||
fa, epoch_row, region_offset, epoch, stage, peer_rank, peer_payload
|
||||
):
|
||||
"""Reset this rank's replica and seed the peer payload needed to finish."""
|
||||
buffer = fa.mnnvl_buffer
|
||||
all_epochs = fa.mnnvl_lamport_epochs
|
||||
assert buffer is not None and all_epochs is not None
|
||||
slot_size = peer_payload.nbytes
|
||||
slot = _stage_slot(
|
||||
buffer, fa.mnnvl_buffer_size, region_offset, stage, peer_rank, slot_size
|
||||
)
|
||||
buffer.view(torch.int32).fill_(_SENTINEL)
|
||||
epochs = all_epochs[epoch_row]
|
||||
# epochs[0] selects the stage, epochs[1] is the CTA completion counter,
|
||||
# and epochs[2:5] record the valid pack count for each stage.
|
||||
epochs.zero_()
|
||||
epochs[0] = epoch
|
||||
epochs[2:5].fill_(slot_size * _NUM_RANKS // _PACK_BYTES)
|
||||
slot.copy_(_payload_bytes(peer_payload))
|
||||
return buffer, slot_size
|
||||
|
||||
|
||||
def _all_ranks_agree(checks, device):
|
||||
"""Give every rank the same result so one failure cannot strand a peer."""
|
||||
values = torch.tensor(checks, dtype=torch.int32, device=device)
|
||||
dist.all_reduce(values, op=dist.ReduceOp.MIN)
|
||||
return [bool(value) for value in values.cpu().tolist()]
|
||||
|
||||
|
||||
def _run_all_gather_retention_case(fa, rank, local_rank, base_epoch, device):
|
||||
slow_epoch = base_epoch
|
||||
fast_epoch = base_epoch + 1
|
||||
slow_stage = slow_epoch % _NUM_STAGES
|
||||
fast_stage = fast_epoch % _NUM_STAGES
|
||||
fast_local = _payload(11, device)
|
||||
fast_peer = _payload(21, device)
|
||||
slow_peer = _payload(31, device)
|
||||
slow_local = _payload(41, device)
|
||||
|
||||
if rank == _FAST_RANK:
|
||||
buffer, slot_size = _prepare_rank_state(
|
||||
fa,
|
||||
_ALL_GATHER_EPOCH_ROW,
|
||||
0,
|
||||
fast_epoch,
|
||||
fast_stage,
|
||||
_SLOW_RANK,
|
||||
fast_peer,
|
||||
)
|
||||
else:
|
||||
buffer, slot_size = _prepare_rank_state(
|
||||
fa,
|
||||
_ALL_GATHER_EPOCH_ROW,
|
||||
0,
|
||||
slow_epoch,
|
||||
slow_stage,
|
||||
_FAST_RANK,
|
||||
slow_peer,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
dist.barrier(device_ids=[local_rank])
|
||||
|
||||
# Fast rank publishes N+1 through the shipping all-gather kernel.
|
||||
fast_output_ok = True
|
||||
if rank == _FAST_RANK:
|
||||
output = fa.custom_all_gather(fast_local)
|
||||
assert output is not None
|
||||
torch.accelerator.synchronize()
|
||||
fast_output_ok = torch.equal(output, torch.cat((fast_local, fast_peer)))
|
||||
dist.barrier(device_ids=[local_rank])
|
||||
|
||||
# Slow rank verifies that publication, then runs shipping epoch-N cleanup.
|
||||
publication_ok = True
|
||||
slow_output_ok = True
|
||||
retained = True
|
||||
if rank == _SLOW_RANK:
|
||||
target = _stage_slot(
|
||||
buffer,
|
||||
fa.mnnvl_buffer_size,
|
||||
0,
|
||||
fast_stage,
|
||||
_FAST_RANK,
|
||||
slot_size,
|
||||
)
|
||||
publication_ok = torch.equal(target, _payload_bytes(fast_local))
|
||||
output = fa.custom_all_gather(slow_local)
|
||||
assert output is not None
|
||||
torch.accelerator.synchronize()
|
||||
slow_output_ok = torch.equal(output, torch.cat((slow_peer, slow_local)))
|
||||
retained = torch.equal(target, _payload_bytes(fast_local))
|
||||
|
||||
checks = _all_ranks_agree(
|
||||
[fast_output_ok, publication_ok, slow_output_ok, retained], device
|
||||
)
|
||||
assert all(checks[:3]), "production all-gather setup did not reach target state"
|
||||
assert checks[3], "all-gather cleanup erased the next-stage publication"
|
||||
|
||||
|
||||
def _run_reduce_scatter_retention_case(fa, rank, local_rank, base_epoch, device):
|
||||
slow_epoch = base_epoch
|
||||
fast_epoch = base_epoch + 1
|
||||
slow_stage = slow_epoch % _NUM_STAGES
|
||||
fast_stage = fast_epoch % _NUM_STAGES
|
||||
region_offset = _REDUCE_SCATTER_REGION_STAGE_OFFSET * fa.mnnvl_buffer_size
|
||||
|
||||
fast_chunk_0 = _payload(11, device)
|
||||
fast_chunk_1 = _payload(12, device)
|
||||
fast_peer = _payload(21, device)
|
||||
slow_chunk_0 = _payload(31, device)
|
||||
slow_chunk_1 = _payload(32, device)
|
||||
slow_peer = _payload(41, device)
|
||||
fast_input = torch.cat((fast_chunk_0, fast_chunk_1))
|
||||
slow_input = torch.cat((slow_chunk_0, slow_chunk_1))
|
||||
|
||||
if rank == _FAST_RANK:
|
||||
buffer, slot_size = _prepare_rank_state(
|
||||
fa,
|
||||
_REDUCE_SCATTER_EPOCH_ROW,
|
||||
region_offset,
|
||||
fast_epoch,
|
||||
fast_stage,
|
||||
_SLOW_RANK,
|
||||
fast_peer,
|
||||
)
|
||||
else:
|
||||
buffer, slot_size = _prepare_rank_state(
|
||||
fa,
|
||||
_REDUCE_SCATTER_EPOCH_ROW,
|
||||
region_offset,
|
||||
slow_epoch,
|
||||
slow_stage,
|
||||
_FAST_RANK,
|
||||
slow_peer,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
dist.barrier(device_ids=[local_rank])
|
||||
|
||||
# Fast rank publishes its N+1 contribution through shipping reduce-scatter.
|
||||
fast_output_ok = True
|
||||
if rank == _FAST_RANK:
|
||||
output = fa.custom_reduce_scatter(fast_input)
|
||||
assert output is not None
|
||||
torch.accelerator.synchronize()
|
||||
fast_output_ok = torch.equal(output, fast_chunk_0 + fast_peer)
|
||||
dist.barrier(device_ids=[local_rank])
|
||||
|
||||
# Slow rank verifies that contribution, then runs shipping epoch-N cleanup.
|
||||
publication_ok = True
|
||||
slow_output_ok = True
|
||||
retained = True
|
||||
if rank == _SLOW_RANK:
|
||||
target = _stage_slot(
|
||||
buffer,
|
||||
fa.mnnvl_buffer_size,
|
||||
region_offset,
|
||||
fast_stage,
|
||||
_FAST_RANK,
|
||||
slot_size,
|
||||
)
|
||||
publication_ok = torch.equal(target, _payload_bytes(fast_chunk_1))
|
||||
output = fa.custom_reduce_scatter(slow_input)
|
||||
assert output is not None
|
||||
torch.accelerator.synchronize()
|
||||
slow_output_ok = torch.equal(output, slow_peer + slow_chunk_1)
|
||||
retained = torch.equal(target, _payload_bytes(fast_chunk_1))
|
||||
|
||||
checks = _all_ranks_agree(
|
||||
[fast_output_ok, publication_ok, slow_output_ok, retained], device
|
||||
)
|
||||
assert all(checks[:3]), "production reduce-scatter setup did not reach target state"
|
||||
assert checks[3], "reduce-scatter cleanup erased the next-stage publication"
|
||||
|
||||
|
||||
def _run_epoch_wrap_case(fa, rank, local_rank, seed_epoch, device):
|
||||
"""Run both shipping collectives across the uint32 wrap boundary."""
|
||||
buffer = fa.mnnvl_buffer
|
||||
all_epochs = fa.mnnvl_lamport_epochs
|
||||
assert buffer is not None and all_epochs is not None
|
||||
buffer.view(torch.int32).fill_(_SENTINEL)
|
||||
all_epochs.zero_()
|
||||
all_epochs[:, 0] = _int32_bits(seed_epoch)
|
||||
torch.accelerator.synchronize()
|
||||
dist.barrier(device_ids=[local_rank])
|
||||
|
||||
ag_input = _payload(rank + 1, device)
|
||||
ag_output = fa.custom_all_gather(ag_input)
|
||||
assert ag_output is not None
|
||||
|
||||
rs_input = torch.cat(
|
||||
(_payload(rank + 1, device), _payload((rank + 1) * 10, device))
|
||||
)
|
||||
rs_output = fa.custom_reduce_scatter(rs_input)
|
||||
assert rs_output is not None
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
next_stage = (seed_epoch % _NUM_STAGES + 1) % _NUM_STAGES
|
||||
checks = _all_ranks_agree(
|
||||
[
|
||||
torch.equal(
|
||||
ag_output,
|
||||
torch.cat((_payload(1, device), _payload(2, device))),
|
||||
),
|
||||
torch.equal(rs_output, _payload(3 if rank == 0 else 30, device)),
|
||||
_epoch_value(all_epochs[_ALL_GATHER_EPOCH_ROW, 0]) == next_stage,
|
||||
_epoch_value(all_epochs[_REDUCE_SCATTER_EPOCH_ROW, 0]) == next_stage,
|
||||
],
|
||||
device,
|
||||
)
|
||||
assert all(checks), "Lamport stages did not advance safely across uint32 wrap"
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def _run_stage_cleanup_test(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
m.setenv("VLLM_ALLREDUCE_USE_SYMM_MEM", "1")
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
|
||||
fa = get_tp_group().device_communicator.ca_comm
|
||||
assert fa is not None and not fa.disabled
|
||||
assert fa.mnnvl_multicast_ptr
|
||||
|
||||
for base_epoch in range(_NUM_STAGES):
|
||||
_run_all_gather_retention_case(fa, rank, rank, base_epoch, device)
|
||||
_run_reduce_scatter_retention_case(fa, rank, rank, base_epoch, device)
|
||||
# Old unbounded metadata stores UINT32_MAX and then repeats stage 0.
|
||||
for seed_epoch in (_UINT32_MAX - 1, _UINT32_MAX):
|
||||
_run_epoch_wrap_case(fa, rank, rank, seed_epoch, device)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _supports_multimem(),
|
||||
reason="MNNVL Lamport collectives require an SM90 or newer NVIDIA GPU.",
|
||||
)
|
||||
def test_mnnvl_lamport_stage_cleanup(monkeypatch: pytest.MonkeyPatch):
|
||||
if torch.accelerator.device_count() < 2:
|
||||
pytest.skip("Need at least two GPUs to run the test.")
|
||||
multi_process_parallel(monkeypatch, 2, 1, _run_stage_cleanup_test)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def _run_multimem_reduce_scatter_test(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
|
||||
fa = get_tp_group().device_communicator.ca_comm
|
||||
assert fa is not None and not fa.disabled
|
||||
assert fa.mnnvl_multimem_rs_supported
|
||||
assert not fa.mnnvl_multimem_rs_initialized
|
||||
assert not fa.mnnvl_multimem_rs_multicast_ptr
|
||||
|
||||
message_bytes = {
|
||||
"mnnvl_lamport": 1 * 1024 * 1024,
|
||||
"mnnvl_multimem": 20 * 1024 * 1024,
|
||||
"nccl": 64 * 1024 * 1024 + 128,
|
||||
}
|
||||
for _ in range(2):
|
||||
for backend, size_bytes in message_bytes.items():
|
||||
shard_elements = size_bytes // torch.bfloat16.itemsize // tp_size
|
||||
inp = torch.full(
|
||||
(tp_size, shard_elements),
|
||||
rank + 1,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
selected = fa._select_reduce_scatter_backend(inp)
|
||||
if backend == "nccl":
|
||||
assert selected is None
|
||||
out = sp_reduce_scatter(inp)
|
||||
else:
|
||||
assert selected == backend
|
||||
out = fa.custom_reduce_scatter(inp)
|
||||
assert out is not None
|
||||
if backend == "mnnvl_multimem":
|
||||
assert fa.mnnvl_multimem_rs_initialized
|
||||
assert fa.mnnvl_multimem_rs_multicast_ptr
|
||||
expected = tp_size * (tp_size + 1) // 2
|
||||
torch.testing.assert_close(out, torch.full_like(out, expected))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _supports_sm100_or_sm103(),
|
||||
reason="The low-SM MNNVL reduce-scatter path requires SM100 or SM103.",
|
||||
)
|
||||
@pytest.mark.parametrize("tp_size", [2, 4, 8])
|
||||
def test_mnnvl_multimem_reduce_scatter(monkeypatch: pytest.MonkeyPatch, tp_size: int):
|
||||
if torch.accelerator.device_count() < tp_size:
|
||||
pytest.skip(f"Need at least {tp_size} GPUs to run the test.")
|
||||
multi_process_parallel(monkeypatch, tp_size, 1, _run_multimem_reduce_scatter_test)
|
||||
@@ -1,290 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..evals.gsm8k.gsm8k_eval import evaluate_gsm8k
|
||||
from ..utils import RemoteOpenAIServer, multi_gpu_test
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cleanup_ray_between_tests():
|
||||
"""Force-stop any lingering Ray processes between tests."""
|
||||
subprocess.run(["ray", "stop", "--force"], timeout=30, capture_output=True)
|
||||
time.sleep(5)
|
||||
yield
|
||||
|
||||
|
||||
MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
||||
|
||||
NUM_GSM8K_QUESTIONS = 256
|
||||
EXPECTED_ACCURACY = 0.58
|
||||
ACCURACY_TOL = 0.08
|
||||
DEFAULT_ALL2ALL_BACKEND = "allgather_reducescatter"
|
||||
|
||||
|
||||
def _send_scale_command(server: RemoteOpenAIServer, new_dp_size: int) -> bool:
|
||||
url = server.url_for("scale_elastic_ep")
|
||||
payload = {"new_data_parallel_size": new_dp_size}
|
||||
headers = {"Content-Type": "application/json"}
|
||||
|
||||
try:
|
||||
response = requests.post(url, json=payload, headers=headers, timeout=300)
|
||||
return response.status_code == 200
|
||||
except requests.exceptions.RequestException:
|
||||
return False
|
||||
|
||||
|
||||
def _traffic_loop(
|
||||
server: RemoteOpenAIServer,
|
||||
dp_rank: int | None,
|
||||
ready: threading.Barrier,
|
||||
stop: threading.Event,
|
||||
finished: threading.Event,
|
||||
is_probe: bool = False,
|
||||
) -> list[tuple[float, float, int | None, str | None]]:
|
||||
url = server.url_for("is_scaling_elastic_ep" if is_probe else "v1/completions")
|
||||
payload = {"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 4}
|
||||
headers = None if dp_rank is None else {"X-data-parallel-rank": str(dp_rank)}
|
||||
request_payload = None if is_probe else payload
|
||||
responses = []
|
||||
is_ready = False
|
||||
while not stop.is_set():
|
||||
request_start = time.perf_counter()
|
||||
phase = None
|
||||
try:
|
||||
response = requests.post(
|
||||
url, json=request_payload, headers=headers, timeout=120
|
||||
)
|
||||
status_code = response.status_code
|
||||
if is_probe and status_code == 200:
|
||||
phase = response.json()["phase"]
|
||||
except requests.exceptions.RequestException:
|
||||
status_code = None
|
||||
responses.append((request_start, time.perf_counter(), status_code, phase))
|
||||
if status_code == 200 and (not is_probe or phase in ("idle", "completed")):
|
||||
if not is_ready:
|
||||
ready.wait(timeout=120)
|
||||
is_ready = True
|
||||
if finished.is_set():
|
||||
return responses
|
||||
time.sleep(0.05)
|
||||
return responses
|
||||
|
||||
|
||||
def _downtime(responses: list[tuple[float, float, int | None, str | None]]) -> float:
|
||||
committing = [end for _, end, _, phase in responses if phase == "committing"]
|
||||
if not committing:
|
||||
return 0
|
||||
recovered = next(
|
||||
end
|
||||
for _, end, status, phase in responses
|
||||
if status == 200 and phase in ("idle", "completed") and end > committing[-1]
|
||||
)
|
||||
return recovered - committing[0]
|
||||
|
||||
|
||||
def _scale_with_traffic(
|
||||
server: RemoteOpenAIServer,
|
||||
source_dp_size: int,
|
||||
new_dp_size: int,
|
||||
traffic_mode: str,
|
||||
) -> None:
|
||||
traffic_clients: list[int | None] = []
|
||||
if traffic_mode == "light":
|
||||
traffic_clients = [0]
|
||||
elif traffic_mode == "heavy":
|
||||
traffic_clients = [None] * source_dp_size
|
||||
clients = [(None, True)] + [(rank, False) for rank in traffic_clients]
|
||||
ready = threading.Barrier(len(clients) + 1)
|
||||
stop = threading.Event()
|
||||
finished = threading.Event()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=len(clients)) as executor:
|
||||
futures = [
|
||||
executor.submit(
|
||||
_traffic_loop, server, rank, ready, stop, finished, is_probe
|
||||
)
|
||||
for rank, is_probe in clients
|
||||
]
|
||||
try:
|
||||
ready.wait(timeout=120)
|
||||
start_time = time.perf_counter()
|
||||
assert _send_scale_command(server, new_dp_size)
|
||||
scale_seconds = time.perf_counter() - start_time
|
||||
finished.set()
|
||||
probe_result, *results = [future.result(timeout=120) for future in futures]
|
||||
finally:
|
||||
stop.set()
|
||||
|
||||
bad_statuses = {
|
||||
status
|
||||
for responses in [probe_result, *results]
|
||||
for _, _, status, _ in responses
|
||||
if status not in (200, 503)
|
||||
}
|
||||
assert not bad_statuses, f"traffic got unexpected statuses {bad_statuses}"
|
||||
probe_committing = [
|
||||
start for start, _, _, phase in probe_result if phase == "committing"
|
||||
]
|
||||
assert probe_committing, "Scaling probe did not observe commit"
|
||||
assert not results or any(
|
||||
status == 200
|
||||
and start_time <= request_start
|
||||
and request_end < probe_committing[0]
|
||||
for responses in results
|
||||
for request_start, request_end, status, _ in responses
|
||||
), "No request completed successfully during preparation"
|
||||
|
||||
print(
|
||||
f"[Elastic EP timing][{source_dp_size}->{new_dp_size}]"
|
||||
f"[traffic={traffic_mode}] "
|
||||
f"scale_seconds={scale_seconds:.3f} "
|
||||
f"downtime_seconds={_downtime(probe_result):.3f}"
|
||||
)
|
||||
|
||||
|
||||
def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
|
||||
assert server.port is not None
|
||||
result = evaluate_gsm8k(
|
||||
num_questions=NUM_GSM8K_QUESTIONS,
|
||||
host=f"http://{server.host}",
|
||||
port=server.port,
|
||||
)
|
||||
accuracy = result["accuracy"]
|
||||
print(
|
||||
f"[{stage}] GSM8K accuracy: {accuracy:.3f} "
|
||||
f"({result['num_questions']} questions)"
|
||||
)
|
||||
assert accuracy >= EXPECTED_ACCURACY, (
|
||||
f"[{stage}] GSM8K accuracy {accuracy:.3f} is below "
|
||||
f"expected threshold {EXPECTED_ACCURACY}"
|
||||
)
|
||||
return accuracy
|
||||
|
||||
|
||||
def _base_serve_args(
|
||||
dp_size: int = 2,
|
||||
max_dp_size: int = 4,
|
||||
enforce_eager: bool = False,
|
||||
all2all_backend: str = DEFAULT_ALL2ALL_BACKEND,
|
||||
) -> list[str]:
|
||||
args = [
|
||||
"--trust-remote-code",
|
||||
"--tensor-parallel-size",
|
||||
"1",
|
||||
"--gpu-memory-utilization",
|
||||
"0.8",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--enable-expert-parallel",
|
||||
"--all2all-backend",
|
||||
all2all_backend,
|
||||
"--enable-elastic-ep",
|
||||
"--elastic-ep-max-dp-size",
|
||||
str(max_dp_size),
|
||||
"--enable-eplb",
|
||||
"--eplb-config.num_redundant_experts",
|
||||
"0",
|
||||
"--eplb-config.use_async",
|
||||
"true",
|
||||
"--eplb-config.step_interval",
|
||||
"300",
|
||||
"--eplb-config.window_size",
|
||||
"5",
|
||||
"--data-parallel-backend",
|
||||
"ray",
|
||||
"--data-parallel-size",
|
||||
str(dp_size),
|
||||
"--api-server-count",
|
||||
"1",
|
||||
"--disable-access-log-for-endpoints",
|
||||
"/is_scaling_elastic_ep",
|
||||
]
|
||||
|
||||
leader_address = os.environ.get("LEADER_ADDRESS")
|
||||
if leader_address:
|
||||
args.extend(["--data-parallel-address", leader_address])
|
||||
if enforce_eager:
|
||||
args.append("--enforce-eager")
|
||||
|
||||
return args
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("enforce_eager", "traffic_mode", "all2all_backend"),
|
||||
[
|
||||
pytest.param(True, "none", DEFAULT_ALL2ALL_BACKEND, id="enforce_eager_none"),
|
||||
pytest.param(True, "light", DEFAULT_ALL2ALL_BACKEND, id="enforce_eager_light"),
|
||||
pytest.param(True, "heavy", DEFAULT_ALL2ALL_BACKEND, id="enforce_eager_heavy"),
|
||||
pytest.param(
|
||||
False,
|
||||
"heavy",
|
||||
DEFAULT_ALL2ALL_BACKEND,
|
||||
id="cuda_graphs_heavy",
|
||||
),
|
||||
pytest.param(False, "heavy", "nixl_ep", id="cuda_graphs_heavy_nixl_ep"),
|
||||
],
|
||||
)
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_elastic_ep_scaling(
|
||||
enforce_eager: bool, traffic_mode: str, all2all_backend: str
|
||||
):
|
||||
from vllm.distributed.eplb.eplb_communicator import has_nixl
|
||||
from vllm.utils.import_utils import has_nixl_ep
|
||||
|
||||
if not has_nixl():
|
||||
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
|
||||
if current_platform.is_xpu():
|
||||
pytest.skip("Elastic EP with NIXL is not supported on XPU")
|
||||
if all2all_backend == "nixl_ep" and not has_nixl_ep():
|
||||
pytest.skip("NIXL EP is not installed")
|
||||
|
||||
initial_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_INITIAL_DP", "2"))
|
||||
target_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_TARGET_DP", "4"))
|
||||
assert target_dp_size > initial_dp_size
|
||||
vllm_serve_args = _base_serve_args(
|
||||
initial_dp_size, target_dp_size, enforce_eager, all2all_backend
|
||||
)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
vllm_serve_args,
|
||||
max_wait_seconds=1200,
|
||||
) as server:
|
||||
initial_accuracy = _run_gsm8k_eval(server, "Initial")
|
||||
|
||||
_scale_with_traffic(server, initial_dp_size, target_dp_size, traffic_mode)
|
||||
scale_up_accuracy = _run_gsm8k_eval(server, "After scale up")
|
||||
assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
|
||||
f"Scale up accuracy {scale_up_accuracy:.3f} dropped more than "
|
||||
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
|
||||
)
|
||||
|
||||
_scale_with_traffic(server, target_dp_size, initial_dp_size, traffic_mode)
|
||||
scale_down_accuracy = _run_gsm8k_eval(server, "After scale down")
|
||||
assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (
|
||||
f"Scale down accuracy {scale_down_accuracy:.3f} dropped more than "
|
||||
f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
|
||||
)
|
||||
|
||||
print("\nAccuracy Summary:")
|
||||
print(f" Initial: {initial_accuracy:.3f}")
|
||||
print(
|
||||
f" Scale up: {scale_up_accuracy:.3f} "
|
||||
f"(diff: {scale_up_accuracy - initial_accuracy:+.3f})"
|
||||
)
|
||||
print(
|
||||
f" Scale down: {scale_down_accuracy:.3f} "
|
||||
f"(diff: {scale_down_accuracy - initial_accuracy:+.3f})"
|
||||
)
|
||||
print(f" Tolerance: {ACCURACY_TOL:.3f}")
|
||||
@@ -1,518 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.eplb.eplb_state import (
|
||||
_compute_eplb_load_stats, # pyright: ignore[reportPrivateUsage]
|
||||
compute_logical_maps,
|
||||
)
|
||||
from vllm.distributed.eplb.policy.default import DefaultEplbPolicy
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_eplb_load_stats_reduce_across_ranks():
|
||||
num_tokens_per_rank = torch.tensor(
|
||||
[
|
||||
[100, 0, 0, 0],
|
||||
[100, 0, 0, 0],
|
||||
],
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
avg_tokens, max_tokens = _compute_eplb_load_stats(num_tokens_per_rank)
|
||||
|
||||
assert avg_tokens.item() == 50
|
||||
assert max_tokens.item() == 200
|
||||
assert (avg_tokens / max_tokens).item() == 0.25
|
||||
|
||||
|
||||
def test_basic_rebalance():
|
||||
"""Test basic rebalancing functionality."""
|
||||
# Example from https://github.com/deepseek-ai/eplb
|
||||
weight = torch.tensor(
|
||||
[
|
||||
[90, 132, 40, 61, 104, 165, 39, 4, 73, 56, 183, 86],
|
||||
[20, 107, 104, 64, 19, 197, 187, 157, 172, 86, 16, 27],
|
||||
]
|
||||
)
|
||||
|
||||
num_layers = weight.shape[0]
|
||||
num_replicas = 16
|
||||
num_groups = 4
|
||||
num_nodes = 2
|
||||
num_gpus = 8
|
||||
|
||||
phy2log = DefaultEplbPolicy.rebalance_experts(
|
||||
weight, num_replicas, num_groups, num_nodes, num_gpus
|
||||
)
|
||||
log2phy, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
|
||||
|
||||
# Verify output shapes
|
||||
assert phy2log.shape == (
|
||||
2,
|
||||
16,
|
||||
), f"Expected `phy2log` shape (2, 16), got {phy2log.shape}"
|
||||
assert log2phy.shape[0] == 2, (
|
||||
f"Expected `log2phy` first dimension 2, got {log2phy.shape[0]}"
|
||||
)
|
||||
assert log2phy.shape[1] == 12, (
|
||||
f"Expected `log2phy` second dimension 12, got {log2phy.shape[1]}"
|
||||
)
|
||||
assert logcnt.shape == (
|
||||
2,
|
||||
12,
|
||||
), f"Expected `logcnt` shape (2, 12), got {logcnt.shape}"
|
||||
|
||||
# Verify physical to logical expert mapping range is correct
|
||||
assert torch.all(phy2log >= 0) and torch.all(phy2log < 12), (
|
||||
"Physical to logical mapping should be in range [0, 12)"
|
||||
)
|
||||
|
||||
# Verify expert count reasonableness
|
||||
assert torch.all(logcnt >= 1), "Each logical expert should have at least 1 replica"
|
||||
assert torch.sum(logcnt, dim=1).sum() == num_replicas * num_layers, (
|
||||
f"Total replicas should be {num_replicas * num_layers}"
|
||||
)
|
||||
|
||||
# Verify expected output
|
||||
expected_phy2log = torch.tensor(
|
||||
[
|
||||
[5, 6, 5, 7, 8, 4, 3, 4, 10, 9, 10, 2, 0, 1, 11, 1],
|
||||
[7, 10, 6, 8, 6, 11, 8, 9, 2, 4, 5, 1, 5, 0, 3, 1],
|
||||
]
|
||||
)
|
||||
assert torch.all(phy2log == expected_phy2log)
|
||||
|
||||
expected_logcnt = torch.tensor(
|
||||
[[1, 2, 1, 1, 2, 2, 1, 1, 1, 1, 2, 1], [1, 2, 1, 1, 1, 2, 2, 1, 2, 1, 1, 1]]
|
||||
)
|
||||
assert torch.all(logcnt == expected_logcnt)
|
||||
|
||||
|
||||
def test_single_gpu_case():
|
||||
"""Test single GPU case."""
|
||||
weight = torch.tensor([[10, 20, 30, 40]])
|
||||
num_replicas = 4
|
||||
num_groups = 1
|
||||
num_nodes = 1
|
||||
num_gpus = 1
|
||||
|
||||
phy2log = DefaultEplbPolicy.rebalance_experts(
|
||||
weight, num_replicas, num_groups, num_nodes, num_gpus
|
||||
)
|
||||
log2phy, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
|
||||
|
||||
# Verify shapes
|
||||
assert phy2log.shape == (1, 4)
|
||||
assert log2phy.shape[0] == 1
|
||||
assert log2phy.shape[1] == 4
|
||||
assert logcnt.shape == (1, 4)
|
||||
|
||||
# Verify all logical experts are mapped
|
||||
assert set(phy2log[0].tolist()) == {0, 1, 2, 3}
|
||||
|
||||
|
||||
def test_equal_weights():
|
||||
"""Test case with equal weights."""
|
||||
weight = torch.tensor([[50, 50, 50, 50, 50, 50, 50, 50]])
|
||||
num_replicas = 8
|
||||
num_groups = 2
|
||||
num_nodes = 2
|
||||
num_gpus = 4
|
||||
|
||||
phy2log = DefaultEplbPolicy.rebalance_experts(
|
||||
weight, num_replicas, num_groups, num_nodes, num_gpus
|
||||
)
|
||||
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
|
||||
|
||||
# Verify shapes
|
||||
assert phy2log.shape == (1, 8)
|
||||
assert logcnt.shape == (1, 8)
|
||||
|
||||
# With equal weights, each expert should have exactly one replica
|
||||
assert torch.all(logcnt == 1), (
|
||||
"With equal weights and no replication, "
|
||||
"each expert should have exactly 1 replica"
|
||||
)
|
||||
|
||||
|
||||
def test_extreme_weight_imbalance():
|
||||
"""Test extreme weight imbalance case."""
|
||||
weight = torch.tensor([[1000, 1, 1, 1, 1, 1, 1, 1]])
|
||||
num_replicas = 12
|
||||
num_groups = 2
|
||||
num_nodes = 2
|
||||
num_gpus = 4
|
||||
|
||||
phy2log = DefaultEplbPolicy.rebalance_experts(
|
||||
weight, num_replicas, num_groups, num_nodes, num_gpus
|
||||
)
|
||||
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
|
||||
|
||||
# Verify shapes
|
||||
assert phy2log.shape == (1, 12)
|
||||
assert logcnt.shape == (1, 8)
|
||||
|
||||
# Expert with highest weight (index 0) should have more replicas
|
||||
assert logcnt[0, 0] > logcnt[0, 1], (
|
||||
"Expert with highest weight should have more replicas"
|
||||
)
|
||||
|
||||
|
||||
def test_multiple_layers():
|
||||
"""Test multiple layers case."""
|
||||
weight = torch.tensor(
|
||||
[
|
||||
[10, 20, 30, 40, 50, 60], # First layer
|
||||
[60, 50, 40, 30, 20, 10], # Second layer (opposite weight pattern)
|
||||
[25, 25, 25, 25, 25, 25], # Third layer (equal weights)
|
||||
]
|
||||
)
|
||||
num_replicas = 8
|
||||
num_groups = 2
|
||||
num_nodes = 2
|
||||
num_gpus = 4
|
||||
|
||||
phy2log = DefaultEplbPolicy.rebalance_experts(
|
||||
weight, num_replicas, num_groups, num_nodes, num_gpus
|
||||
)
|
||||
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
|
||||
|
||||
# Verify shapes
|
||||
assert phy2log.shape == (3, 8)
|
||||
assert logcnt.shape == (3, 6)
|
||||
|
||||
# Verify expert allocation is reasonable for each layer
|
||||
for layer in range(3):
|
||||
assert torch.all(phy2log[layer] >= 0) and torch.all(phy2log[layer] < 6), (
|
||||
f"Layer {layer} physical to logical mappingshould be in range [0, 6)"
|
||||
)
|
||||
assert torch.sum(logcnt[layer]) == num_replicas, (
|
||||
f"Layer {layer} total replicas should be {num_replicas}"
|
||||
)
|
||||
|
||||
|
||||
def test_parameter_validation():
|
||||
"""Test parameter validation."""
|
||||
weight = torch.tensor([[10, 20, 30, 40]])
|
||||
|
||||
# Test non-divisible case - this should handle normally without throwing
|
||||
# errors because the function will fall back to global load balancing
|
||||
# strategy
|
||||
phy2log = DefaultEplbPolicy.rebalance_experts(weight, 8, 3, 2, 4)
|
||||
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
|
||||
assert phy2log.shape == (1, 8)
|
||||
assert logcnt.shape == (1, 4)
|
||||
|
||||
# Test cases that will actually cause errors:
|
||||
# num_physical_experts not divisible by num_gpus
|
||||
with pytest.raises(AssertionError):
|
||||
DefaultEplbPolicy.rebalance_experts(weight, 7, 2, 2, 4) # 7 not divisible by 4
|
||||
|
||||
|
||||
def test_small_scale_hierarchical():
|
||||
"""Test small-scale hierarchical load balancing."""
|
||||
weight = torch.tensor(
|
||||
[
|
||||
[100, 50, 200, 75, 150, 25, 300, 80], # 8 experts
|
||||
]
|
||||
)
|
||||
num_replicas = 12
|
||||
num_groups = 4 # 4 groups, 2 experts each
|
||||
num_nodes = 2 # 2 nodes
|
||||
num_gpus = 4 # 4 GPUs
|
||||
|
||||
phy2log = DefaultEplbPolicy.rebalance_experts(
|
||||
weight, num_replicas, num_groups, num_nodes, num_gpus
|
||||
)
|
||||
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
|
||||
|
||||
# Verify basic constraints
|
||||
assert phy2log.shape == (1, 12)
|
||||
assert logcnt.shape == (1, 8)
|
||||
assert torch.sum(logcnt) == num_replicas
|
||||
assert torch.all(logcnt >= 1)
|
||||
|
||||
# Expert with highest weight should have more replicas
|
||||
max_weight_expert = torch.argmax(weight[0])
|
||||
assert logcnt[0, max_weight_expert] >= 2, (
|
||||
"Highest weight expert should have multiple replicas"
|
||||
)
|
||||
|
||||
|
||||
def test_global_load_balance_fallback():
|
||||
"""Test global load balancing fallback case."""
|
||||
# When num_groups % num_nodes != 0, should fall back to global load
|
||||
# balancing
|
||||
weight = torch.tensor([[10, 20, 30, 40, 50, 60]])
|
||||
num_replicas = 8
|
||||
num_groups = 3 # Cannot be divided evenly by num_nodes=2
|
||||
num_nodes = 2
|
||||
num_gpus = 4
|
||||
|
||||
phy2log = DefaultEplbPolicy.rebalance_experts(
|
||||
weight, num_replicas, num_groups, num_nodes, num_gpus
|
||||
)
|
||||
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
|
||||
|
||||
# Should work normally, just using global load balancing strategy
|
||||
assert phy2log.shape == (1, 8)
|
||||
assert logcnt.shape == (1, 6)
|
||||
assert torch.sum(logcnt) == num_replicas
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda", "xpu"])
|
||||
def test_device_compatibility(device):
|
||||
"""Test device compatibility."""
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA not available")
|
||||
if device == "xpu" and not current_platform.is_xpu():
|
||||
pytest.skip("XPU not available")
|
||||
|
||||
weight = torch.tensor([[10, 20, 30, 40]], device=device)
|
||||
num_replicas = 6
|
||||
num_groups = 2
|
||||
num_nodes = 1
|
||||
num_gpus = 2
|
||||
|
||||
phy2log = DefaultEplbPolicy.rebalance_experts(
|
||||
weight, num_replicas, num_groups, num_nodes, num_gpus
|
||||
)
|
||||
_, logcnt = compute_logical_maps(phy2log, weight.shape[-1])
|
||||
|
||||
# Function will convert to CPU internally, but should handle different
|
||||
# device inputs normally
|
||||
assert phy2log.shape == (1, 6)
|
||||
assert logcnt.shape == (1, 4)
|
||||
|
||||
|
||||
def test_additional_cases():
|
||||
"""Test more edge cases and different parameter combinations."""
|
||||
# Test case 1: Large-scale distributed setup
|
||||
weight1 = torch.tensor(
|
||||
[[50, 100, 75, 120, 90, 60, 80, 110, 40, 70, 95, 85, 65, 55, 45, 35]]
|
||||
)
|
||||
phy2log1 = DefaultEplbPolicy.rebalance_experts(weight1, 24, 8, 4, 8)
|
||||
_, logcnt1 = compute_logical_maps(phy2log1, weight1.shape[-1])
|
||||
|
||||
assert phy2log1.shape == (1, 24)
|
||||
assert logcnt1.shape == (1, 16)
|
||||
assert torch.sum(logcnt1) == 24
|
||||
|
||||
# Test case 2: Different weight distributions
|
||||
weight2 = torch.tensor(
|
||||
[
|
||||
[200, 150, 100, 50, 25, 12], # Decreasing weights
|
||||
[12, 25, 50, 100, 150, 200], # Increasing weights
|
||||
]
|
||||
)
|
||||
phy2log2 = DefaultEplbPolicy.rebalance_experts(weight2, 10, 3, 1, 2)
|
||||
_, logcnt2 = compute_logical_maps(phy2log2, weight2.shape[-1])
|
||||
|
||||
assert phy2log2.shape == (2, 10)
|
||||
assert logcnt2.shape == (2, 6)
|
||||
|
||||
# Verify high-weight experts have more replicas
|
||||
for layer in range(2):
|
||||
max_weight_idx = torch.argmax(weight2[layer])
|
||||
assert logcnt2[layer, max_weight_idx] >= 2
|
||||
|
||||
|
||||
def test_compute_logical_maps_with_negative_indices():
|
||||
"""Test that compute_logical_maps correctly handles physical slots containing
|
||||
-1 (unused slots).
|
||||
"""
|
||||
# 2 layers, 6 physical slots, 4 logical experts.
|
||||
# Slots 2 and 5 are unused (-1).
|
||||
phy2log = torch.tensor(
|
||||
[
|
||||
[0, 1, -1, 2, 3, -1],
|
||||
[3, -1, 2, 1, 0, -1],
|
||||
]
|
||||
)
|
||||
num_layers = 2
|
||||
num_logical_experts = 4
|
||||
|
||||
log2phy, logcnt = compute_logical_maps(phy2log, num_logical_experts)
|
||||
|
||||
assert logcnt.shape == (num_layers, num_logical_experts)
|
||||
assert log2phy.shape == (num_layers, num_logical_experts, 1)
|
||||
|
||||
expected_logcnt = torch.ones(num_layers, num_logical_experts, dtype=phy2log.dtype)
|
||||
assert torch.all(logcnt == expected_logcnt), (
|
||||
f"Expected that all replica counts == 1, got {logcnt}"
|
||||
)
|
||||
|
||||
assert torch.all(log2phy >= 0), (
|
||||
"log2phy should only contain valid physical indices, not -1"
|
||||
)
|
||||
|
||||
assert log2phy[0, 0, 0] == 0
|
||||
assert log2phy[0, 1, 0] == 1
|
||||
assert log2phy[0, 2, 0] == 3
|
||||
assert log2phy[0, 3, 0] == 4
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
weight = torch.tensor(
|
||||
[
|
||||
[90, 132, 40, 61, 104, 165, 39, 4, 73, 56, 183, 86],
|
||||
[20, 107, 104, 64, 19, 197, 187, 157, 172, 86, 16, 27],
|
||||
]
|
||||
)
|
||||
|
||||
num_replicas = 16
|
||||
num_groups = 4
|
||||
num_nodes = 2
|
||||
num_gpus = 8
|
||||
|
||||
phy2log = DefaultEplbPolicy.rebalance_experts(
|
||||
weight, num_replicas, num_groups, num_nodes, num_gpus
|
||||
)
|
||||
print(phy2log)
|
||||
|
||||
test_basic_rebalance()
|
||||
|
||||
|
||||
def _make_phy_replicas_idx_from_phy2log(phy2log: np.ndarray) -> np.ndarray:
|
||||
"""Create replicas indices mapping from phy2log."""
|
||||
pr = np.zeros_like(phy2log, dtype=np.int64)
|
||||
for layer in range(phy2log.shape[0]):
|
||||
seen: dict[int, int] = {}
|
||||
row = phy2log[layer].tolist()
|
||||
for i, expert in enumerate(row):
|
||||
r = seen.get(expert, 0)
|
||||
pr[layer, i] = r
|
||||
seen[expert] = r + 1
|
||||
return pr
|
||||
|
||||
|
||||
def _validate_intragpu_rearrangement(
|
||||
old_global_expert_indices: np.ndarray,
|
||||
new_phy2log: np.ndarray,
|
||||
new_phy_replicas_idx: np.ndarray,
|
||||
post_phy2log: np.ndarray,
|
||||
post_phy_replicas_idx: np.ndarray,
|
||||
num_ranks: int,
|
||||
slots_per_gpu: int,
|
||||
):
|
||||
# Per-GPU checks
|
||||
for gpu_idx in range(num_ranks):
|
||||
start = gpu_idx * slots_per_gpu
|
||||
end = start + slots_per_gpu
|
||||
old_seg = old_global_expert_indices[0, start:end]
|
||||
new_seg = new_phy2log[0, start:end]
|
||||
new_rnk = new_phy_replicas_idx[0, start:end]
|
||||
post_seg = post_phy2log[0, start:end]
|
||||
post_rnk = post_phy_replicas_idx[0, start:end]
|
||||
|
||||
# Pairwise equality for (expert, rank) pairs to ensure nothing is lost
|
||||
def sorted_pairs(seg, rnk):
|
||||
pairs = list(zip(seg.tolist(), rnk.tolist()))
|
||||
pairs.sort()
|
||||
return pairs
|
||||
|
||||
assert sorted_pairs(post_seg, post_rnk) == sorted_pairs(new_seg, new_rnk), (
|
||||
f"Per-GPU pairs of (expert,rank) must match new mapping for GPU {gpu_idx}"
|
||||
)
|
||||
|
||||
# For experts that remain on the same GPU, the old slot is preserved
|
||||
# for at least one occurrence; rank at that slot must be valid for that expert
|
||||
old_list = old_seg.tolist()
|
||||
new_list = new_seg.tolist()
|
||||
post_list = post_seg.tolist()
|
||||
remained = set(old_list) & set(new_list)
|
||||
new_ranks_for_expert: dict[int, list[int]] = {}
|
||||
for v, r in zip(new_list, new_rnk.tolist()):
|
||||
new_ranks_for_expert.setdefault(v, []).append(r)
|
||||
for expert in remained:
|
||||
old_pos = old_list.index(expert)
|
||||
assert post_list[old_pos] == expert, (
|
||||
f"Expert {expert} on GPU {gpu_idx} should stay at old slot {old_pos}"
|
||||
)
|
||||
# Rank at preserved slot must be one of the ranks
|
||||
# the expert has in new mapping
|
||||
assert post_rnk.tolist()[old_pos] in new_ranks_for_expert[expert], (
|
||||
f"Rank for expert {expert} at preserved slot on GPU {gpu_idx} "
|
||||
"must come from new mapping"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_ranks, slots_per_gpu, old_phy2log, new_phy2log",
|
||||
[
|
||||
pytest.param(
|
||||
# Setup: 2 GPUs, 4 slots each, 1 layer
|
||||
# Old mapping: GPU0 -> [0,1,2,3], GPU1 -> [4,5,6,7]
|
||||
# New mapping shuffles within GPU0 and brings 4,5 into GPU0.
|
||||
# GPU0 new -> [1,5,0,4]; GPU1 new -> [6,2,7,3]
|
||||
2,
|
||||
4,
|
||||
np.array([[0, 1, 2, 3, 4, 5, 6, 7]]),
|
||||
np.array([[1, 5, 0, 4, 6, 2, 7, 3]]),
|
||||
id="simple",
|
||||
),
|
||||
pytest.param(
|
||||
# Setup: 2 GPUs, 5 slots each (total 10 physical experts), 1 layer
|
||||
# Old mapping:
|
||||
# GPU0 -> [0, 1, 0, 2, 3] (expert 0 duplicated)
|
||||
# GPU1 -> [4, 5, 6, 1, 2]
|
||||
# New mapping reorders within GPUs and moves some experts across GPUs,
|
||||
# while still including duplicates:
|
||||
# GPU0 new -> [0, 5, 4, 0, 1] (expert 0 duplicated, 4/5 incoming)
|
||||
# GPU1 new -> [6, 2, 3, 2, 1] (expert 2 duplicated)
|
||||
2,
|
||||
5,
|
||||
np.array([[0, 1, 0, 2, 3, 4, 5, 6, 1, 2]]),
|
||||
np.array([[0, 5, 4, 0, 1, 6, 2, 3, 2, 1]]),
|
||||
id="duplicates",
|
||||
),
|
||||
pytest.param(
|
||||
# Setup: 3 GPUs, 4 slots each (total 12 physical experts), 1 layer
|
||||
# Old mapping:
|
||||
# GPU0 -> [0, 1, 2, 3]
|
||||
# GPU1 -> [0, 1, 2, 3]
|
||||
# GPU2 -> [0, 1, 2, 3]
|
||||
# New mapping decides to use one expert on 2 GPUs and shuffles
|
||||
# experts on the third GPU,
|
||||
# GPU0 new -> [0, 0, 0, 0]
|
||||
# GPU1 new -> [0, 0, 0, 0]
|
||||
# GPU2 new -> [1, 2, 3, 0]
|
||||
3,
|
||||
4,
|
||||
np.array([[0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3]]),
|
||||
np.array([[0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 0]]),
|
||||
id="skewed_expert",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_preserve_intragpu_slots(
|
||||
num_ranks: int,
|
||||
slots_per_gpu: int,
|
||||
old_phy2log: torch.Tensor,
|
||||
new_phy2log: torch.Tensor,
|
||||
):
|
||||
"""Experts that stay on a GPU keep their old slots; incoming not lost."""
|
||||
phy_replicas_idx = _make_phy_replicas_idx_from_phy2log(new_phy2log)
|
||||
|
||||
post_phy2log = DefaultEplbPolicy.preserve_intragpu_slots(
|
||||
new_phy2log, num_ranks, old_phy2log
|
||||
)
|
||||
post_phy_replicas_idx = _make_phy_replicas_idx_from_phy2log(post_phy2log)
|
||||
|
||||
# Shapes preserved
|
||||
assert post_phy2log.shape == new_phy2log.shape
|
||||
assert post_phy_replicas_idx.shape == phy_replicas_idx.shape
|
||||
|
||||
_validate_intragpu_rearrangement(
|
||||
old_phy2log,
|
||||
new_phy2log,
|
||||
phy_replicas_idx,
|
||||
post_phy2log,
|
||||
post_phy_replicas_idx,
|
||||
num_ranks,
|
||||
slots_per_gpu,
|
||||
)
|
||||
@@ -1,104 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_wait_blocks_until_record():
|
||||
event = CpuGpuEvent()
|
||||
record_stream = torch.Stream()
|
||||
wait_stream = torch.Stream()
|
||||
wait_returned = threading.Event()
|
||||
|
||||
def waiter():
|
||||
event.wait(stream=wait_stream)
|
||||
wait_returned.set()
|
||||
|
||||
t = threading.Thread(target=waiter)
|
||||
t.start()
|
||||
|
||||
time.sleep(0.05)
|
||||
assert not wait_returned.is_set(), "wait() returned before record() was called"
|
||||
|
||||
event.record(stream=record_stream)
|
||||
t.join(timeout=5.0)
|
||||
|
||||
assert not event._recorded.is_set()
|
||||
|
||||
|
||||
def test_reuse_across_multiple_cycles():
|
||||
wrapper = CpuGpuEvent()
|
||||
record_stream = torch.Stream()
|
||||
wait_stream = torch.Stream()
|
||||
NUM_CYCLES = 8
|
||||
completed_cycles = []
|
||||
barriers = [threading.Barrier(2) for _ in range(NUM_CYCLES)]
|
||||
|
||||
def waiter():
|
||||
for i in range(NUM_CYCLES):
|
||||
wrapper.wait(stream=wait_stream)
|
||||
completed_cycles.append(True)
|
||||
barriers[i].wait()
|
||||
|
||||
t = threading.Thread(target=waiter)
|
||||
t.start()
|
||||
|
||||
for i in range(NUM_CYCLES):
|
||||
wrapper.record(stream=record_stream)
|
||||
barriers[i].wait()
|
||||
|
||||
t.join(timeout=10.0)
|
||||
assert len(completed_cycles) == NUM_CYCLES
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cuda", "xpu"])
|
||||
def test_producer_consumer(device):
|
||||
"""This test uses the CpuGpuEvent to synchronize reads and writes to/from
|
||||
a shared GPU tensor on multiple CPU threads.
|
||||
"""
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA not available")
|
||||
if device == "xpu" and not current_platform.is_xpu():
|
||||
pytest.skip("XPU not available")
|
||||
worker_stream = torch.Stream()
|
||||
# Create a single element counter that will be shared between two threads
|
||||
buf = torch.zeros(1, device=device)
|
||||
NUM_ROUNDS = 5
|
||||
|
||||
ready_cpu = [threading.Event() for _ in range(NUM_ROUNDS)]
|
||||
events = [CpuGpuEvent() for _ in range(NUM_ROUNDS)]
|
||||
errors: list[str] = []
|
||||
|
||||
# For each round, the worker thread (writer) sets the counter in buf and waits for
|
||||
# the main thread to read it.
|
||||
def worker():
|
||||
for i in range(NUM_ROUNDS):
|
||||
if i > 0:
|
||||
events[i - 1].wait(stream=worker_stream)
|
||||
|
||||
with device_stream(worker_stream):
|
||||
buf.fill_(float(i + 1))
|
||||
|
||||
worker_stream.synchronize()
|
||||
ready_cpu[i].set()
|
||||
|
||||
t = threading.Thread(target=worker)
|
||||
t.start()
|
||||
|
||||
for i in range(NUM_ROUNDS):
|
||||
ready_cpu[i].wait()
|
||||
snapshot = buf.clone()
|
||||
events[i].record()
|
||||
val = snapshot.item()
|
||||
if val != float(i + 1):
|
||||
errors.append(f"round {i}: expected {i + 1:.1f}, got {val:.1f}")
|
||||
|
||||
t.join(timeout=10.0)
|
||||
assert not errors, f"Buffer ordering errors: {errors}"
|
||||
@@ -1,894 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import random
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed
|
||||
|
||||
import vllm.distributed.eplb.eplb_communicator as eplb_comm
|
||||
import vllm.utils.gpu_sync_debug as gsd
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed.eplb.eplb_communicator import (
|
||||
create_eplb_communicator,
|
||||
has_nixl,
|
||||
)
|
||||
from vllm.distributed.eplb.rebalance_execute import (
|
||||
move_from_buffer,
|
||||
rearrange_expert_weights_inplace,
|
||||
transfer_layer,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .eplb_utils import distributed_run, set_env_vars_and_device
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_gloo_receive_staging_does_not_force_gpu_sync(monkeypatch):
|
||||
"""Gloo receives must reach the GPU without an implicit pageable copy."""
|
||||
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error")
|
||||
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
|
||||
gsd._install_copy_checkers()
|
||||
monkeypatch.setattr(eplb_comm, "is_local_first_rank", lambda: False)
|
||||
|
||||
monkeypatch.setattr(
|
||||
eplb_comm,
|
||||
"P2POp",
|
||||
lambda op, tensor, peer=None, group=None, tag=0, group_peer=None: tensor,
|
||||
)
|
||||
|
||||
def receive(tensors):
|
||||
for tensor in tensors:
|
||||
tensor.fill_(7)
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(eplb_comm, "batch_isend_irecv", receive)
|
||||
communicator = eplb_comm.TorchDistGlooStagedEplbCommunicator(cpu_group=None)
|
||||
dst = torch.empty(32, device="cuda")
|
||||
communicator.add_recv([dst], src_rank=1, expert_id=0)
|
||||
gsd.with_gpu_sync_check(communicator.execute)()
|
||||
torch.testing.assert_close(dst.cpu(), torch.full((32,), 7.0))
|
||||
|
||||
|
||||
def create_expert_indices_with_redundancy(
|
||||
num_layers: int,
|
||||
num_logical_experts: int,
|
||||
total_physical_experts: int,
|
||||
redundancy_config: list[int], # redundancy for each logical expert
|
||||
) -> torch.Tensor:
|
||||
"""Create expert indices with redundancy.
|
||||
|
||||
Args:
|
||||
num_layers: number of layers
|
||||
num_logical_experts: number of logical experts
|
||||
total_physical_experts: total number of physical experts
|
||||
redundancy_config: redundancy for each logical expert
|
||||
|
||||
Returns:
|
||||
indices: Shape (num_layers, total_physical_experts)
|
||||
|
||||
"""
|
||||
assert sum(redundancy_config) == total_physical_experts
|
||||
assert len(redundancy_config) == num_logical_experts
|
||||
|
||||
indices = torch.zeros(num_layers, total_physical_experts, dtype=torch.long)
|
||||
|
||||
for layer in range(num_layers):
|
||||
physical_pos = 0
|
||||
for logical_expert_id, redundancy in enumerate(redundancy_config):
|
||||
for _ in range(redundancy):
|
||||
indices[layer, physical_pos] = logical_expert_id
|
||||
physical_pos += 1
|
||||
|
||||
# Shuffle the indices at dim 1
|
||||
for layer in range(num_layers):
|
||||
indices[layer] = indices[layer][torch.randperm(indices.shape[1])]
|
||||
|
||||
return indices
|
||||
|
||||
|
||||
def create_expert_weights(
|
||||
num_layers: int,
|
||||
num_local_experts: int,
|
||||
hidden_sizes: list[int],
|
||||
rank: int,
|
||||
device: torch.device,
|
||||
physical_to_logical_mapping: torch.Tensor,
|
||||
) -> list[list[torch.Tensor]]:
|
||||
"""Create fake expert weights tensor for testing.
|
||||
|
||||
Use `arange` to generate predictable weights values, based on logical
|
||||
expert ID.
|
||||
All replicas of the same logical expert should have the same weights.
|
||||
|
||||
Args:
|
||||
physical_to_logical_mapping: Shape (num_layers, num_local_experts)
|
||||
mapping[layer, physical_pos] = logical_expert_id
|
||||
|
||||
"""
|
||||
expert_weights = []
|
||||
|
||||
for layer in range(num_layers):
|
||||
layer_weights = []
|
||||
for weight_idx, hidden_size in enumerate(hidden_sizes):
|
||||
weight_tensor = torch.zeros(
|
||||
num_local_experts, hidden_size, device=device, dtype=torch.float32
|
||||
)
|
||||
|
||||
for local_expert in range(num_local_experts):
|
||||
# Get the logical expert ID for this physical expert
|
||||
global_pos = rank * num_local_experts + local_expert
|
||||
logical_expert_id = physical_to_logical_mapping[
|
||||
layer, global_pos
|
||||
].item()
|
||||
|
||||
# Generate weights based on logical expert ID
|
||||
# (so that all replicas of the same logical expert have the
|
||||
# same weights)
|
||||
base_value = logical_expert_id * 1000 + layer * 100 + weight_idx * 10
|
||||
weight_tensor[local_expert] = torch.arange(
|
||||
base_value,
|
||||
base_value + hidden_size,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
layer_weights.append(weight_tensor)
|
||||
expert_weights.append(layer_weights)
|
||||
|
||||
return expert_weights
|
||||
|
||||
|
||||
def create_redundancy_config(
|
||||
num_logical_experts: int,
|
||||
num_physical_experts: int,
|
||||
) -> list[int]:
|
||||
"""Create a redundancy configuration."""
|
||||
redundancy_config = [1] * num_logical_experts
|
||||
remaining = num_physical_experts - num_logical_experts
|
||||
# Randomly assign the remaining physical experts to the logical experts
|
||||
for _ in range(remaining):
|
||||
redundancy_config[random.choice(range(num_logical_experts))] += 1
|
||||
return redundancy_config
|
||||
|
||||
|
||||
def verify_expert_weights_after_shuffle(
|
||||
expert_weights: list[list[torch.Tensor]],
|
||||
new_indices: torch.Tensor,
|
||||
hidden_sizes: list[int],
|
||||
ep_rank: int,
|
||||
num_local_experts: int,
|
||||
) -> bool:
|
||||
"""Verify the weights after shuffling are correct."""
|
||||
num_layers = len(expert_weights)
|
||||
ok = True
|
||||
|
||||
for layer in range(num_layers):
|
||||
for weight_idx, hidden_size in enumerate(hidden_sizes):
|
||||
weight_tensor = expert_weights[layer][weight_idx]
|
||||
|
||||
for local_expert in range(num_local_experts):
|
||||
# Calculate the global expert ID for this local expert
|
||||
global_pos = ep_rank * num_local_experts + local_expert
|
||||
expected_logical_expert = new_indices[layer, global_pos].item()
|
||||
|
||||
# Check if the weights are correct
|
||||
actual_weights = weight_tensor[local_expert]
|
||||
expected_base = (
|
||||
expected_logical_expert * 1000 + layer * 100 + weight_idx * 10
|
||||
)
|
||||
expected_weights = torch.arange(
|
||||
expected_base,
|
||||
expected_base + hidden_size,
|
||||
device=actual_weights.device,
|
||||
dtype=actual_weights.dtype,
|
||||
)
|
||||
|
||||
if not torch.equal(actual_weights, expected_weights):
|
||||
ok = False
|
||||
actual_head = actual_weights[:8].detach().cpu().tolist()
|
||||
expected_head = expected_weights[:8].detach().cpu().tolist()
|
||||
print(
|
||||
"verify_expert_weights_after_shuffle failed: "
|
||||
f"rank={ep_rank}, "
|
||||
f"layer={layer}, weight_idx={weight_idx}, "
|
||||
f"local_expert={local_expert}, "
|
||||
f"expected_logical_expert={expected_logical_expert}, "
|
||||
f"actual_head={actual_head}, expected_head={expected_head}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
return ok
|
||||
|
||||
|
||||
def verify_redundant_experts_have_same_weights(
|
||||
expert_weights: list[list[torch.Tensor]],
|
||||
indices: torch.Tensor,
|
||||
hidden_sizes: list[int],
|
||||
ep_rank: int,
|
||||
world_size: int,
|
||||
num_local_experts: int,
|
||||
cpu_group,
|
||||
) -> bool:
|
||||
"""Verify that all replicas of the same logical expert have the same weights."""
|
||||
num_layers = len(expert_weights)
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
# XPU: gloo does not support XPU tensors, so gather on CPU then move back.
|
||||
# CUDA: gloo supports CUDA tensors directly, keep original device.
|
||||
use_cpu_gather = current_platform.is_xpu()
|
||||
|
||||
ok = True
|
||||
for layer in range(num_layers):
|
||||
# Collect weights for all physical experts for each weight matrix
|
||||
all_weights: list[torch.Tensor] = []
|
||||
|
||||
for weight_idx, hidden_size in enumerate(hidden_sizes):
|
||||
orig_device = expert_weights[layer][weight_idx].device
|
||||
|
||||
if use_cpu_gather:
|
||||
local_weights = expert_weights[layer][weight_idx].cpu()
|
||||
gather_device = "cpu"
|
||||
else:
|
||||
local_weights = expert_weights[layer][weight_idx]
|
||||
gather_device = orig_device
|
||||
|
||||
# Create tensor to store all expert weights
|
||||
# Shape: [total_physical_experts, hidden_size]
|
||||
gathered_weights = torch.zeros(
|
||||
total_physical_experts,
|
||||
hidden_size,
|
||||
device=gather_device,
|
||||
dtype=local_weights.dtype,
|
||||
)
|
||||
|
||||
# Split tensor along dim 0 into a list for all_gather
|
||||
gathered_weights_list = torch.chunk(gathered_weights, world_size, dim=0)
|
||||
|
||||
torch.distributed.all_gather(
|
||||
list(gathered_weights_list),
|
||||
local_weights,
|
||||
group=cpu_group,
|
||||
)
|
||||
|
||||
if use_cpu_gather:
|
||||
gathered_weights = gathered_weights.to(orig_device)
|
||||
all_weights.append(gathered_weights)
|
||||
|
||||
# Verify that all replicas of the same logical expert have the same
|
||||
# weights
|
||||
logical_expert_weights: dict[int, dict[int, torch.Tensor]] = {}
|
||||
|
||||
for physical_pos in range(total_physical_experts):
|
||||
logical_expert_id = int(indices[layer, physical_pos].item())
|
||||
|
||||
if logical_expert_id not in logical_expert_weights:
|
||||
# First time encountering this logical expert, save its weights
|
||||
logical_expert_weights[logical_expert_id] = {
|
||||
weight_idx: all_weights[weight_idx][physical_pos]
|
||||
for weight_idx in range(len(hidden_sizes))
|
||||
}
|
||||
else:
|
||||
# Verify that current physical expert's weights match the
|
||||
# previously saved logical expert weights
|
||||
for weight_idx in range(len(hidden_sizes)):
|
||||
if not torch.equal(
|
||||
all_weights[weight_idx][physical_pos],
|
||||
logical_expert_weights[logical_expert_id][weight_idx],
|
||||
):
|
||||
ok = False
|
||||
actual_head = (
|
||||
all_weights[weight_idx][physical_pos][:8]
|
||||
.detach()
|
||||
.cpu()
|
||||
.tolist()
|
||||
)
|
||||
reference_head = (
|
||||
logical_expert_weights[logical_expert_id][weight_idx][:8]
|
||||
.detach()
|
||||
.cpu()
|
||||
.tolist()
|
||||
)
|
||||
print(
|
||||
"verify_redundant_experts_have_same_weights failed: "
|
||||
f"rank={ep_rank}, "
|
||||
f"layer={layer}, weight_idx={weight_idx}, "
|
||||
f"logical_expert={logical_expert_id}, "
|
||||
f"physical_pos={physical_pos}, "
|
||||
f"actual_head={actual_head}, "
|
||||
f"reference_head={reference_head}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
return ok
|
||||
|
||||
|
||||
def assert_verification_synced(local_ok: bool, msg: str, cpu_group) -> None:
|
||||
if current_platform.is_xpu():
|
||||
# XPU: default group backend (gloo) does not support XPU tensors;
|
||||
# use CPU tensor with cpu_group instead.
|
||||
ok_tensor = torch.tensor(
|
||||
[1 if local_ok else 0], device="cpu", dtype=torch.int32
|
||||
)
|
||||
torch.distributed.all_reduce(
|
||||
ok_tensor, op=torch.distributed.ReduceOp.MIN, group=cpu_group
|
||||
)
|
||||
else:
|
||||
# CUDA: nccl (default group) supports device tensors directly.
|
||||
device = torch.accelerator.current_accelerator()
|
||||
ok_tensor = torch.tensor(
|
||||
[1 if local_ok else 0], device=device, dtype=torch.int32
|
||||
)
|
||||
torch.distributed.all_reduce(ok_tensor, op=torch.distributed.ReduceOp.MIN)
|
||||
assert bool(ok_tensor.item()), msg
|
||||
|
||||
|
||||
def create_eplb_communicator_or_raise(
|
||||
*, group_coordinator, backend, expert_weights, expert_buffer
|
||||
):
|
||||
try:
|
||||
return create_eplb_communicator(
|
||||
group_coordinator=group_coordinator,
|
||||
backend=backend,
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"Failed to create EPLB communicator for backend={backend}: {exc}"
|
||||
) from exc
|
||||
|
||||
|
||||
def _test_async_transfer_layer_without_mtp_worker(
|
||||
env,
|
||||
world_size: int,
|
||||
num_layers: int,
|
||||
num_local_experts: int,
|
||||
num_logical_experts: int,
|
||||
eplb_communicator: str,
|
||||
) -> None:
|
||||
set_env_vars_and_device(env)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.tensor_parallel_size = world_size
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
hidden_sizes = [16, 32]
|
||||
|
||||
redundancy_config = create_redundancy_config(
|
||||
num_logical_experts,
|
||||
total_physical_experts,
|
||||
)
|
||||
old_indices = create_expert_indices_with_redundancy(
|
||||
num_layers,
|
||||
num_logical_experts,
|
||||
total_physical_experts,
|
||||
redundancy_config,
|
||||
)
|
||||
|
||||
new_redundancy_config = create_redundancy_config(
|
||||
num_logical_experts,
|
||||
total_physical_experts,
|
||||
)
|
||||
new_indices = create_expert_indices_with_redundancy(
|
||||
num_layers,
|
||||
num_logical_experts,
|
||||
total_physical_experts,
|
||||
new_redundancy_config,
|
||||
)
|
||||
|
||||
expert_weights = create_expert_weights(
|
||||
num_layers,
|
||||
num_local_experts,
|
||||
hidden_sizes,
|
||||
ep_rank,
|
||||
device,
|
||||
old_indices,
|
||||
)
|
||||
old_indices_cpu = old_indices.cpu()
|
||||
new_indices_cpu = new_indices.cpu()
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
stream = torch.Stream(device=device)
|
||||
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend=eplb_communicator,
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
communicator.set_stream(stream)
|
||||
|
||||
for layer_idx in range(num_layers):
|
||||
transfer_metadata = transfer_layer(
|
||||
old_layer_indices=old_indices_cpu[layer_idx],
|
||||
new_layer_indices=new_indices_cpu[layer_idx],
|
||||
expert_weights=expert_weights[layer_idx],
|
||||
expert_weights_buffer=expert_buffer,
|
||||
ep_group=ep_group,
|
||||
communicator=communicator,
|
||||
stream=stream,
|
||||
layer_idx=layer_idx,
|
||||
)
|
||||
stream.synchronize()
|
||||
move_from_buffer(
|
||||
expert_weights=expert_weights[layer_idx],
|
||||
expert_weights_buffers=expert_buffer,
|
||||
transfer_metadata=transfer_metadata,
|
||||
new_indices=new_indices_cpu[layer_idx].numpy(),
|
||||
ep_rank=ep_rank,
|
||||
)
|
||||
|
||||
local_ok = verify_expert_weights_after_shuffle(
|
||||
expert_weights,
|
||||
new_indices,
|
||||
hidden_sizes,
|
||||
ep_rank,
|
||||
num_local_experts,
|
||||
)
|
||||
local_ok = (
|
||||
verify_redundant_experts_have_same_weights(
|
||||
expert_weights,
|
||||
new_indices,
|
||||
hidden_sizes,
|
||||
ep_rank,
|
||||
world_size,
|
||||
num_local_experts,
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
and local_ok
|
||||
)
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"Async transfer verification failed on at least one rank. "
|
||||
"See logs for details.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
def _test_rearrange_expert_weights_with_redundancy(
|
||||
env,
|
||||
world_size,
|
||||
num_layers,
|
||||
num_local_experts,
|
||||
num_logical_experts,
|
||||
eplb_communicator: str,
|
||||
) -> None:
|
||||
# Initialize model parallel (using tensor parallel as an entrypoint
|
||||
# to expert parallel)
|
||||
set_env_vars_and_device(env)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.tensor_parallel_size = world_size
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
# Test parameters
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
hidden_sizes = [32, 64] # Two different weight matrices
|
||||
|
||||
# Create old expert indices (with redundancy)
|
||||
redundancy_config = create_redundancy_config(
|
||||
num_logical_experts, total_physical_experts
|
||||
)
|
||||
|
||||
old_indices = create_expert_indices_with_redundancy(
|
||||
num_layers,
|
||||
num_logical_experts,
|
||||
total_physical_experts,
|
||||
redundancy_config,
|
||||
)
|
||||
|
||||
# Create new expert indices (with redundancy)
|
||||
new_redundancy_config = create_redundancy_config(
|
||||
num_logical_experts, total_physical_experts
|
||||
)
|
||||
new_indices = create_expert_indices_with_redundancy(
|
||||
num_layers,
|
||||
num_logical_experts,
|
||||
total_physical_experts,
|
||||
new_redundancy_config,
|
||||
)
|
||||
|
||||
# Create expert weights
|
||||
expert_weights = create_expert_weights(
|
||||
num_layers, num_local_experts, hidden_sizes, ep_rank, device, old_indices
|
||||
)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend=eplb_communicator,
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
|
||||
# Execute weight rearrangement
|
||||
rearrange_expert_weights_inplace(
|
||||
old_indices,
|
||||
new_indices,
|
||||
expert_weights,
|
||||
expert_buffer,
|
||||
ep_group,
|
||||
communicator,
|
||||
)
|
||||
|
||||
# Verify the rearrangement result
|
||||
local_ok = verify_expert_weights_after_shuffle(
|
||||
expert_weights,
|
||||
new_indices,
|
||||
hidden_sizes,
|
||||
ep_rank,
|
||||
num_local_experts,
|
||||
)
|
||||
|
||||
local_ok = (
|
||||
verify_redundant_experts_have_same_weights(
|
||||
expert_weights,
|
||||
new_indices,
|
||||
hidden_sizes,
|
||||
ep_rank,
|
||||
world_size,
|
||||
num_local_experts,
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
and local_ok
|
||||
)
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"Rearrange verification failed on at least one rank. See logs for details.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"world_size,num_layers,num_local_experts,num_logical_experts",
|
||||
[
|
||||
# 2 GPU, 2 experts per GPU
|
||||
# 3 logical experts, 4 physical experts, 1 redundant experts
|
||||
(2, 1, 2, 3),
|
||||
# 2 GPU, 3 experts per GPU
|
||||
# 4 logical experts, 6 physical experts, 2 redundant experts
|
||||
(2, 2, 3, 4),
|
||||
# 2 GPU, 8 experts per GPU
|
||||
# 16 logical experts, 16 physical experts, 0 redundant experts
|
||||
(2, 4, 8, 16),
|
||||
# 4 GPU, 2 experts per GPU
|
||||
# 6 logical experts, 8 physical experts, 2 redundant experts
|
||||
(4, 1, 2, 6),
|
||||
# 4 GPU, 2 experts per GPU
|
||||
# 5 logical experts, 8 physical experts, 3 redundant experts
|
||||
(4, 2, 2, 5),
|
||||
# 4 GPU, 8 experts per GPU
|
||||
# 16 logical experts, 32 physical experts, 16 redundant experts
|
||||
(4, 8, 8, 16),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"eplb_communicator", ["torch_nccl", "torch_gloo", "torch_xccl", "pynccl", "nixl"]
|
||||
)
|
||||
def test_rearrange_expert_weights_with_redundancy(
|
||||
world_size,
|
||||
num_layers,
|
||||
num_local_experts,
|
||||
num_logical_experts,
|
||||
eplb_communicator,
|
||||
):
|
||||
"""Test the functionality of rearranging expert weights with redundancy."""
|
||||
if eplb_communicator == "nixl" and not has_nixl():
|
||||
pytest.skip("NIXL is not available")
|
||||
if eplb_communicator == "nixl" and current_platform.is_xpu():
|
||||
pytest.skip("NIXL does not support XPU")
|
||||
if eplb_communicator in ("torch_nccl", "pynccl") and not torch.cuda.is_available():
|
||||
pytest.skip(f"{eplb_communicator} requires CUDA")
|
||||
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
|
||||
pytest.skip("torch_xccl requires XPU")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(
|
||||
_test_rearrange_expert_weights_with_redundancy,
|
||||
world_size,
|
||||
num_layers,
|
||||
num_local_experts,
|
||||
num_logical_experts,
|
||||
eplb_communicator,
|
||||
)
|
||||
|
||||
|
||||
def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
set_env_vars_and_device(env)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.tensor_parallel_size = world_size
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
num_layers = 2
|
||||
num_local_experts = 2
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
num_logical_experts = total_physical_experts // 2 # Some redundancy
|
||||
hidden_sizes = [32, 64]
|
||||
|
||||
# Create redundancy configuration
|
||||
redundancy_config = [2] * num_logical_experts
|
||||
|
||||
# Same indices - no change
|
||||
indices = create_expert_indices_with_redundancy(
|
||||
num_layers, num_logical_experts, total_physical_experts, redundancy_config
|
||||
)
|
||||
|
||||
expert_weights = create_expert_weights(
|
||||
num_layers, num_local_experts, hidden_sizes, ep_rank, device, indices
|
||||
)
|
||||
|
||||
# Save original weights
|
||||
original_weights = []
|
||||
for layer_weights in expert_weights:
|
||||
layer_copy = []
|
||||
for weight in layer_weights:
|
||||
layer_copy.append(weight.clone())
|
||||
original_weights.append(layer_copy)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend=default_backend,
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
|
||||
# Execute rearrangement (should be no change)
|
||||
rearrange_expert_weights_inplace(
|
||||
indices,
|
||||
indices, # Same indices
|
||||
expert_weights,
|
||||
expert_buffer,
|
||||
ep_group,
|
||||
communicator,
|
||||
)
|
||||
|
||||
# Verify that the weights have not changed
|
||||
local_ok = True
|
||||
for layer in range(num_layers):
|
||||
for weight_idx in range(len(hidden_sizes)):
|
||||
if not torch.equal(
|
||||
expert_weights[layer][weight_idx],
|
||||
original_weights[layer][weight_idx],
|
||||
):
|
||||
local_ok = False
|
||||
print(
|
||||
"test_rearrange_expert_weights_no_change failed: "
|
||||
f"layer={layer}, weight_idx={weight_idx}",
|
||||
flush=True,
|
||||
)
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"No-change EPLB verification failed on at least one rank.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"world_size,num_layers,num_local_experts,num_logical_experts",
|
||||
[
|
||||
(2, 2, 2, 3),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("eplb_communicator", ["torch_gloo", "torch_xccl", "nixl"])
|
||||
def test_async_transfer_layer_without_mtp(
|
||||
world_size: int,
|
||||
num_layers: int,
|
||||
num_local_experts: int,
|
||||
num_logical_experts: int,
|
||||
eplb_communicator: str,
|
||||
):
|
||||
"""Exercise async EPLB transfer path without MTP/spec decode."""
|
||||
if eplb_communicator == "nixl" and not has_nixl():
|
||||
pytest.skip("NIXL is not available")
|
||||
if eplb_communicator == "nixl" and current_platform.is_xpu():
|
||||
pytest.skip("NIXL does not support XPU")
|
||||
if eplb_communicator == "torch_xccl" and not current_platform.is_xpu():
|
||||
pytest.skip("torch_xccl requires XPU")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
|
||||
distributed_run(
|
||||
_test_async_transfer_layer_without_mtp_worker,
|
||||
world_size,
|
||||
num_layers,
|
||||
num_local_experts,
|
||||
num_logical_experts,
|
||||
eplb_communicator,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_rearrange_expert_weights_no_change(world_size):
|
||||
"""Test that when the indices do not change, the weights should remain
|
||||
unchanged.
|
||||
"""
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(
|
||||
_test_rearrange_expert_weights_no_change,
|
||||
world_size,
|
||||
)
|
||||
|
||||
|
||||
def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
set_env_vars_and_device(env)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.tensor_parallel_size = world_size
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
|
||||
)
|
||||
|
||||
ep_group_coordinator = get_tp_group()
|
||||
ep_group = ep_group_coordinator.device_group
|
||||
cpu_group = ep_group_coordinator.cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
accelerator_type = torch.accelerator.current_accelerator().type
|
||||
device = torch.device(f"{accelerator_type}:{ep_rank}")
|
||||
|
||||
num_layers = 1
|
||||
num_local_experts = 2
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
num_logical_experts = total_physical_experts // 2
|
||||
hidden_sizes = [32]
|
||||
|
||||
# Create different index distributions
|
||||
old_redundancy = create_redundancy_config(
|
||||
num_logical_experts, total_physical_experts
|
||||
)
|
||||
new_redundancy = create_redundancy_config(
|
||||
num_logical_experts, total_physical_experts
|
||||
)
|
||||
|
||||
old_indices = create_expert_indices_with_redundancy(
|
||||
num_layers, num_logical_experts, total_physical_experts, old_redundancy
|
||||
)
|
||||
new_indices = create_expert_indices_with_redundancy(
|
||||
num_layers, num_logical_experts, total_physical_experts, new_redundancy
|
||||
)
|
||||
|
||||
expert_weights = create_expert_weights(
|
||||
num_layers, num_local_experts, hidden_sizes, ep_rank, device, old_indices
|
||||
)
|
||||
|
||||
# Save original weights
|
||||
original_weights = []
|
||||
for layer_weights in expert_weights:
|
||||
layer_copy = []
|
||||
for weight in layer_weights:
|
||||
layer_copy.append(weight.clone())
|
||||
original_weights.append(layer_copy)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend=default_backend,
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
|
||||
# Execute profile mode rearrangement
|
||||
rearrange_expert_weights_inplace(
|
||||
old_indices,
|
||||
new_indices,
|
||||
expert_weights,
|
||||
expert_buffer,
|
||||
ep_group,
|
||||
communicator,
|
||||
is_profile=True,
|
||||
)
|
||||
|
||||
# In profile mode, the weights should remain unchanged
|
||||
local_ok = True
|
||||
for layer in range(num_layers):
|
||||
for weight_idx in range(len(hidden_sizes)):
|
||||
if not torch.equal(
|
||||
expert_weights[layer][weight_idx],
|
||||
original_weights[layer][weight_idx],
|
||||
):
|
||||
local_ok = False
|
||||
print(
|
||||
"test_rearrange_expert_weights_profile_mode failed: "
|
||||
f"layer={layer}, weight_idx={weight_idx}",
|
||||
flush=True,
|
||||
)
|
||||
assert_verification_synced(
|
||||
local_ok,
|
||||
"Profile-mode EPLB verification failed on at least one rank.",
|
||||
cpu_group=cpu_group,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_rearrange_expert_weights_profile_mode(world_size):
|
||||
"""Test profile mode (should not copy actual weights)"""
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(
|
||||
_test_rearrange_expert_weights_profile_mode,
|
||||
world_size,
|
||||
)
|
||||
|
||||
|
||||
def _test_eplb_communicator_pipeline_parallel(env, world_size, backend):
|
||||
set_env_vars_and_device(env)
|
||||
config = VllmConfig()
|
||||
config.parallel_config.tensor_parallel_size = 2
|
||||
config.parallel_config.pipeline_parallel_size = world_size // 2
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=2,
|
||||
pipeline_model_parallel_size=world_size // 2,
|
||||
)
|
||||
ep_group = get_tp_group()
|
||||
peer_rank = 1 - ep_group.rank_in_group
|
||||
rank = torch.distributed.get_rank()
|
||||
send_tensor = torch.full((16,), rank, device="cuda", dtype=torch.int32)
|
||||
recv_tensor = torch.empty_like(send_tensor)
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group,
|
||||
backend=backend,
|
||||
expert_weights=[[send_tensor]],
|
||||
expert_buffer=[recv_tensor],
|
||||
)
|
||||
communicator.add_send([send_tensor], peer_rank, expert_id=0)
|
||||
communicator.add_recv([recv_tensor], peer_rank, expert_id=0)
|
||||
communicator.execute()
|
||||
|
||||
expected = torch.full_like(recv_tensor, ep_group.ranks[peer_rank])
|
||||
torch.testing.assert_close(recv_tensor, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
@pytest.mark.parametrize("backend", ["torch_nccl", "torch_gloo"])
|
||||
def test_eplb_communicator_pipeline_parallel(world_size, backend):
|
||||
"""Transfer weights within each PP stage, including nonzero global ranks."""
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(_test_eplb_communicator_pipeline_parallel, world_size, backend)
|
||||
@@ -1,298 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Test that the interaction between EPLB and FusedMoEFactory Layer is okay
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
|
||||
from vllm.distributed.eplb.rebalance_execute import rearrange_expert_weights_inplace
|
||||
from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
get_eplb_group,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .eplb_utils import distributed_run, set_env_vars_and_device
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestConfig:
|
||||
num_layers: int
|
||||
num_experts: int
|
||||
num_local_experts: int
|
||||
num_topk: int
|
||||
hidden_size: int
|
||||
intermediate_size: int
|
||||
weight_dtype: torch.dtype
|
||||
weight_scale_dtype: torch.dtype | None
|
||||
column_major_scales: bool
|
||||
|
||||
|
||||
def make_expert_weights(
|
||||
layer_idx: int,
|
||||
global_expert_idx: int,
|
||||
global_num_experts: int,
|
||||
tensor_shape: tuple[int, ...],
|
||||
tensor_dtype: torch.dtype,
|
||||
tensor_device: torch.device,
|
||||
is_column_major: bool,
|
||||
) -> torch.Tensor:
|
||||
assert len(tensor_shape) == 2
|
||||
|
||||
if is_column_major:
|
||||
tensor_shape = (tensor_shape[1], tensor_shape[0])
|
||||
|
||||
x = torch.empty(tensor_shape, dtype=tensor_dtype, device=tensor_device)
|
||||
value_offset = (layer_idx * global_num_experts + global_expert_idx) * x.numel()
|
||||
x.view(-1).copy_(
|
||||
torch.arange(
|
||||
value_offset,
|
||||
value_offset + x.numel(),
|
||||
dtype=tensor_dtype,
|
||||
device=tensor_device,
|
||||
)
|
||||
)
|
||||
|
||||
if is_column_major:
|
||||
x = torch.transpose(x, 1, 0)
|
||||
assert not x.is_contiguous()
|
||||
return x
|
||||
|
||||
|
||||
def make_fused_moe_layer(
|
||||
rank: int,
|
||||
layer_idx: int,
|
||||
test_config: TestConfig,
|
||||
) -> MoERunner:
|
||||
fml = FusedMoEFactory(
|
||||
num_experts=test_config.num_experts,
|
||||
top_k=test_config.num_topk,
|
||||
hidden_size=test_config.hidden_size,
|
||||
intermediate_size=test_config.intermediate_size,
|
||||
prefix=f"dummy_layer_{layer_idx}",
|
||||
activation="silu",
|
||||
params_dtype=test_config.weight_dtype,
|
||||
)
|
||||
re = fml.routed_experts
|
||||
|
||||
device = torch.device(f"{torch.accelerator.current_accelerator().type}:{rank}")
|
||||
|
||||
from functools import partial
|
||||
|
||||
_make_expert_weights = partial(
|
||||
make_expert_weights,
|
||||
layer_idx=layer_idx,
|
||||
global_num_experts=test_config.num_experts,
|
||||
tensor_device=device,
|
||||
)
|
||||
|
||||
assert isinstance(re.w13_weight.data, torch.Tensor)
|
||||
assert isinstance(re.w2_weight.data, torch.Tensor)
|
||||
re.w13_weight.data = re.w13_weight.data.to(device=device)
|
||||
re.w2_weight.data = re.w2_weight.data.to(device=device)
|
||||
w13_weight = re.w13_weight.data
|
||||
w2_weight = re.w2_weight.data
|
||||
assert w13_weight.size(0) == test_config.num_local_experts
|
||||
for i in range(test_config.num_local_experts):
|
||||
g_i = rank * test_config.num_local_experts + i
|
||||
w13_weight_e = w13_weight[i]
|
||||
w2_weight_e = w2_weight[i]
|
||||
w13_weight_e.copy_(
|
||||
_make_expert_weights(
|
||||
global_expert_idx=g_i,
|
||||
tensor_shape=w13_weight_e.shape,
|
||||
tensor_dtype=w13_weight_e.dtype,
|
||||
is_column_major=False,
|
||||
)
|
||||
)
|
||||
w2_weight_e.copy_(
|
||||
_make_expert_weights(
|
||||
global_expert_idx=g_i,
|
||||
tensor_shape=w2_weight_e.shape,
|
||||
tensor_dtype=w2_weight_e.dtype,
|
||||
is_column_major=False,
|
||||
)
|
||||
)
|
||||
|
||||
block_size = 16
|
||||
|
||||
def block_quant_scales_shape(
|
||||
shape: tuple[int, ...], is_column_major: bool
|
||||
) -> tuple[int, ...]:
|
||||
assert len(shape) == 3
|
||||
if not is_column_major:
|
||||
return (shape[0], shape[1] // block_size, shape[2] // block_size)
|
||||
else:
|
||||
return (shape[0], shape[2] // block_size, shape[1] // block_size)
|
||||
|
||||
is_column_major = test_config.column_major_scales
|
||||
w13_weight_scale_inv = torch.empty(
|
||||
block_quant_scales_shape(w13_weight.shape, is_column_major),
|
||||
dtype=test_config.weight_dtype,
|
||||
device=device,
|
||||
)
|
||||
w2_weight_scale_inv = torch.empty(
|
||||
block_quant_scales_shape(w2_weight.shape, is_column_major),
|
||||
dtype=test_config.weight_dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
for i in range(test_config.num_local_experts):
|
||||
g_i = rank * test_config.num_local_experts + i
|
||||
w13_s_e = w13_weight_scale_inv[i]
|
||||
w2_s_e = w2_weight_scale_inv[i]
|
||||
w13_s_e.copy_(
|
||||
_make_expert_weights(
|
||||
global_expert_idx=g_i,
|
||||
tensor_shape=w13_s_e.shape,
|
||||
tensor_dtype=w13_s_e.dtype,
|
||||
# Fill data in row-major and then
|
||||
# transpose if test_config requires col-major.
|
||||
is_column_major=False,
|
||||
)
|
||||
)
|
||||
w2_s_e.copy_(
|
||||
_make_expert_weights(
|
||||
global_expert_idx=g_i,
|
||||
tensor_shape=w2_s_e.shape,
|
||||
tensor_dtype=w2_s_e.dtype,
|
||||
is_column_major=False,
|
||||
)
|
||||
)
|
||||
if is_column_major:
|
||||
w13_weight_scale_inv = torch.transpose(w13_weight_scale_inv, 1, 2)
|
||||
w2_weight_scale_inv = torch.transpose(w2_weight_scale_inv, 1, 2)
|
||||
assert not w13_weight_scale_inv.is_contiguous()
|
||||
assert not w2_weight_scale_inv.is_contiguous()
|
||||
|
||||
# Add scales to the parameter list
|
||||
re.w13_weight_scale_inv = torch.nn.Parameter(
|
||||
w13_weight_scale_inv, requires_grad=False
|
||||
)
|
||||
re.w2_weight_scale_inv = torch.nn.Parameter(
|
||||
w2_weight_scale_inv, requires_grad=False
|
||||
)
|
||||
|
||||
return fml
|
||||
|
||||
|
||||
def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
|
||||
# Initialize model parallel (using tensor parallel as an entrypoint
|
||||
# to expert parallel)
|
||||
set_env_vars_and_device(env)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.tensor_parallel_size = world_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = True
|
||||
vllm_config.parallel_config.enable_eplb = True
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
|
||||
)
|
||||
|
||||
ep_group = get_tp_group().device_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
|
||||
fml_layers = [
|
||||
make_fused_moe_layer(ep_rank, layer_idx, test_config)
|
||||
for layer_idx in range(test_config.num_layers)
|
||||
]
|
||||
rank_expert_weights = [fml.get_expert_weights() for fml in fml_layers]
|
||||
|
||||
indices = torch.zeros(
|
||||
test_config.num_layers, test_config.num_experts, dtype=torch.long
|
||||
)
|
||||
for lidx in range(test_config.num_layers):
|
||||
indices[lidx] = torch.Tensor(range(test_config.num_experts))
|
||||
|
||||
shuffled_indices = torch.zeros_like(indices)
|
||||
for lidx in range(test_config.num_layers):
|
||||
shuffled_indices[lidx] = torch.randperm(test_config.num_experts)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in rank_expert_weights[0]]
|
||||
default_backend = "torch_xccl" if current_platform.is_xpu() else "torch_nccl"
|
||||
communicator = create_eplb_communicator(
|
||||
group_coordinator=get_eplb_group(),
|
||||
backend=default_backend,
|
||||
expert_weights=rank_expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
rearrange_expert_weights_inplace(
|
||||
indices,
|
||||
shuffled_indices,
|
||||
rank_expert_weights,
|
||||
expert_buffer,
|
||||
ep_group,
|
||||
communicator,
|
||||
)
|
||||
|
||||
num_local_experts = test_config.num_local_experts
|
||||
num_global_experts = test_config.num_experts
|
||||
for lidx, fml in enumerate(fml_layers):
|
||||
for name, w in fml.named_parameters():
|
||||
for e in range(num_local_experts):
|
||||
g_e = shuffled_indices[lidx][ep_rank * num_local_experts + e]
|
||||
ref = make_expert_weights(
|
||||
layer_idx=lidx,
|
||||
global_expert_idx=int(g_e.item()),
|
||||
global_num_experts=num_global_experts,
|
||||
tensor_shape=w[e].shape,
|
||||
tensor_dtype=w[e].dtype,
|
||||
tensor_device=w[e].device,
|
||||
is_column_major=not w[e].is_contiguous(),
|
||||
)
|
||||
assert w[e].shape == ref.shape and w[e].stride() == ref.stride(), (
|
||||
f"w[{e}] {w[e].size()} {w[e].stride()} vs "
|
||||
f"ref {ref.size()} {ref.stride()}"
|
||||
)
|
||||
torch.testing.assert_close(w[e], ref)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("world_size", [2])
|
||||
@pytest.mark.parametrize("num_layers", [4])
|
||||
@pytest.mark.parametrize("num_experts", [16])
|
||||
@pytest.mark.parametrize("hidden_size", [256])
|
||||
@pytest.mark.parametrize("intermediate_size", [256])
|
||||
@pytest.mark.parametrize("column_major_scales", [True, False])
|
||||
def test_eplb_fml(
|
||||
world_size: int,
|
||||
num_layers: int,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
column_major_scales: bool,
|
||||
):
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
|
||||
num_local_experts = num_experts // world_size
|
||||
num_topk = 4
|
||||
# The dtypes are fine as we are essentially just checking data-copies
|
||||
weight_dtype = torch.bfloat16
|
||||
weight_scale_dtype = torch.bfloat16
|
||||
|
||||
test_config = TestConfig(
|
||||
num_layers=num_layers,
|
||||
num_experts=num_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
num_topk=num_topk,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
weight_dtype=weight_dtype,
|
||||
weight_scale_dtype=weight_scale_dtype,
|
||||
column_major_scales=column_major_scales,
|
||||
)
|
||||
|
||||
distributed_run(
|
||||
_test_eplb_fml,
|
||||
world_size,
|
||||
test_config,
|
||||
)
|
||||
@@ -1,295 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Test that the interaction between EPLB and MoERunner Layer is okay for DP w/ NVFP4
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import make_test_quant_config
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.distributed.eplb.rebalance_execute import rearrange_expert_weights_inplace
|
||||
from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
get_dp_group,
|
||||
get_eplb_group,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory, MoERunner
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
ModelOptNvFp4FusedMoE,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .eplb_utils import distributed_run, set_env_vars_and_device
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestConfig:
|
||||
num_layers: int
|
||||
num_experts: int
|
||||
num_local_experts: int
|
||||
num_topk: int
|
||||
hidden_size: int
|
||||
intermediate_size: int
|
||||
num_tokens: int
|
||||
moe_backend: str
|
||||
|
||||
|
||||
def make_fused_moe_layer(
|
||||
rank: int,
|
||||
layer_idx: int,
|
||||
test_config: TestConfig,
|
||||
) -> MoERunner:
|
||||
quant_config = None
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
|
||||
quant_config = ModelOptNvFp4Config(
|
||||
is_checkpoint_nvfp4_serialized=True,
|
||||
kv_cache_quant_algo=None,
|
||||
exclude_modules=[],
|
||||
)
|
||||
|
||||
fml = FusedMoEFactory(
|
||||
num_experts=test_config.num_experts,
|
||||
top_k=test_config.num_topk,
|
||||
hidden_size=test_config.hidden_size,
|
||||
intermediate_size=test_config.intermediate_size,
|
||||
prefix=f"dummy_layer_{layer_idx}",
|
||||
activation="silu",
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
nvfp4_fused_moe = ModelOptNvFp4FusedMoE(quant_config, fml)
|
||||
nvfp4_fused_moe.create_weights(
|
||||
fml,
|
||||
test_config.num_local_experts,
|
||||
test_config.hidden_size,
|
||||
test_config.intermediate_size,
|
||||
params_dtype=torch.uint8,
|
||||
global_num_experts=test_config.num_experts,
|
||||
)
|
||||
|
||||
fml = fml.to(device)
|
||||
re = fml.routed_experts
|
||||
w1_q, w2_q, quant_config = make_test_quant_config(
|
||||
test_config.num_local_experts,
|
||||
test_config.intermediate_size,
|
||||
test_config.hidden_size,
|
||||
in_dtype=torch.bfloat16,
|
||||
quant_dtype="nvfp4",
|
||||
block_shape=None,
|
||||
per_act_token_quant=False,
|
||||
)
|
||||
|
||||
re.w13_weight.data = w1_q
|
||||
re.w2_weight.data = w2_q
|
||||
|
||||
re.w2_input_scale.data = torch.randn_like(re.w2_input_scale.data) / 5
|
||||
re.w13_input_scale.data = torch.randn_like(re.w13_input_scale.data) / 5
|
||||
re.w2_weight_scale_2.data = torch.randn_like(re.w2_weight_scale_2.data) / 5
|
||||
re.w13_weight_scale_2.data = torch.randn_like(re.w13_weight_scale_2.data) / 5
|
||||
re.w2_weight_scale.data = (
|
||||
torch.randn(re.w2_weight_scale.data.shape, device=device) / 5
|
||||
).to(re.w2_weight_scale.data.dtype)
|
||||
re.w13_weight_scale.data = (
|
||||
torch.randn(re.w13_weight_scale.data.shape, device=device) / 5
|
||||
).to(re.w13_weight_scale.data.dtype)
|
||||
|
||||
nvfp4_fused_moe.process_weights_after_loading(re)
|
||||
|
||||
fml.maybe_init_modular_kernel()
|
||||
|
||||
return fml
|
||||
|
||||
|
||||
def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
|
||||
set_env_vars_and_device(env)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.data_parallel_size = world_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = True
|
||||
vllm_config.kernel_config.moe_backend = test_config.moe_backend
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=1, pipeline_model_parallel_size=1
|
||||
)
|
||||
|
||||
ep_group = get_dp_group().cpu_group
|
||||
ep_rank = torch.distributed.get_rank()
|
||||
|
||||
device = torch.device(f"cuda:{ep_rank}")
|
||||
|
||||
fml_layers = [
|
||||
make_fused_moe_layer(ep_rank, layer_idx, test_config).to(device)
|
||||
for layer_idx in range(test_config.num_layers)
|
||||
]
|
||||
rank_expert_weights = [fml.get_expert_weights() for fml in fml_layers]
|
||||
|
||||
hidden_states = []
|
||||
router_logits = []
|
||||
for layer_idx in range(test_config.num_layers):
|
||||
hidden_states.append(
|
||||
torch.randn(
|
||||
(test_config.num_tokens, test_config.hidden_size),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
)
|
||||
router_logits.append(
|
||||
torch.randn(
|
||||
(test_config.num_tokens, test_config.num_experts),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
)
|
||||
|
||||
out_before_shuffle = []
|
||||
with set_forward_context(
|
||||
{},
|
||||
num_tokens=test_config.num_tokens,
|
||||
num_tokens_across_dp=torch.tensor(
|
||||
[test_config.num_tokens] * world_size, device="cpu", dtype=torch.int
|
||||
),
|
||||
vllm_config=vllm_config,
|
||||
):
|
||||
for lidx, fml in enumerate(fml_layers):
|
||||
out_before_shuffle.append(
|
||||
fml(hidden_states[lidx].clone(), router_logits[lidx].clone())
|
||||
)
|
||||
|
||||
indices = torch.zeros(
|
||||
test_config.num_layers, test_config.num_experts, dtype=torch.long
|
||||
)
|
||||
for lidx in range(test_config.num_layers):
|
||||
indices[lidx] = torch.Tensor(range(test_config.num_experts))
|
||||
|
||||
shuffled_indices = torch.zeros_like(indices)
|
||||
for lidx in range(test_config.num_layers):
|
||||
shuffled_indices[lidx] = torch.randperm(test_config.num_experts)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in rank_expert_weights[0]]
|
||||
communicator = create_eplb_communicator(
|
||||
group_coordinator=get_eplb_group(),
|
||||
backend="torch_nccl",
|
||||
expert_weights=rank_expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
rearrange_expert_weights_inplace(
|
||||
indices,
|
||||
shuffled_indices,
|
||||
rank_expert_weights,
|
||||
expert_buffer,
|
||||
ep_group,
|
||||
communicator,
|
||||
)
|
||||
|
||||
num_global_experts = test_config.num_experts
|
||||
|
||||
logical_to_physical_map_list = []
|
||||
for lidx, fml in enumerate(fml_layers):
|
||||
physical_to_logical_map = shuffled_indices[lidx].to(device)
|
||||
logical_to_physical_map = torch.empty(
|
||||
(num_global_experts,), dtype=torch.int32, device=device
|
||||
)
|
||||
logical_to_physical_map[physical_to_logical_map] = torch.arange(
|
||||
0, num_global_experts, dtype=torch.int32, device=device
|
||||
)
|
||||
logical_to_physical_map_list.append(
|
||||
logical_to_physical_map.reshape(num_global_experts, 1)
|
||||
)
|
||||
|
||||
logical_to_physical_map = torch.stack(logical_to_physical_map_list)
|
||||
|
||||
for lidx, fml in enumerate(fml_layers):
|
||||
logical_replica_count = torch.ones(
|
||||
(test_config.num_layers, num_global_experts),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
fml.eplb_state = EplbLayerState()
|
||||
fml.set_eplb_state(
|
||||
lidx,
|
||||
torch.zeros(
|
||||
(test_config.num_layers, num_global_experts),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
),
|
||||
logical_to_physical_map,
|
||||
logical_replica_count,
|
||||
)
|
||||
fml.router.eplb_state.should_record_tensor = torch.ones(
|
||||
(), dtype=torch.bool, device=device
|
||||
)
|
||||
fml.router.eplb_state.num_unpadded_tokens_tensors = [
|
||||
torch.tensor(0, dtype=torch.int32, device=device)
|
||||
]
|
||||
|
||||
out_after_shuffle = []
|
||||
with set_forward_context(
|
||||
{},
|
||||
num_tokens=test_config.num_tokens,
|
||||
num_tokens_across_dp=torch.tensor(
|
||||
[test_config.num_tokens] * world_size, device="cpu", dtype=torch.int
|
||||
),
|
||||
vllm_config=vllm_config,
|
||||
):
|
||||
for lidx, fml in enumerate(fml_layers):
|
||||
out_after_shuffle.append(
|
||||
fml(hidden_states[lidx].clone(), router_logits[lidx].clone())
|
||||
)
|
||||
|
||||
for lidx in range(test_config.num_layers):
|
||||
torch.testing.assert_close(
|
||||
out_before_shuffle[lidx], out_after_shuffle[lidx], atol=1e-1, rtol=1e-1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
@pytest.mark.parametrize("num_layers", [8])
|
||||
@pytest.mark.parametrize("num_experts", [32])
|
||||
@pytest.mark.parametrize("hidden_size", [256])
|
||||
@pytest.mark.parametrize("intermediate_size", [256])
|
||||
@pytest.mark.parametrize("num_tokens", [256])
|
||||
@pytest.mark.parametrize("moe_backend", ["flashinfer_trtllm", "flashinfer_cutlass"])
|
||||
def test_eplb_fml(
|
||||
world_size: int,
|
||||
num_layers: int,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
num_tokens: int,
|
||||
moe_backend: str,
|
||||
):
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
if current_platform.is_xpu():
|
||||
pytest.skip("NVFP4 quantization and flashinfer backends are CUDA-only")
|
||||
|
||||
num_local_experts = num_experts // world_size
|
||||
num_topk = 4
|
||||
|
||||
test_config = TestConfig(
|
||||
num_layers=num_layers,
|
||||
num_experts=num_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
num_topk=num_topk,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
num_tokens=num_tokens,
|
||||
moe_backend=moe_backend,
|
||||
)
|
||||
|
||||
distributed_run(
|
||||
_test_eplb_fml,
|
||||
world_size,
|
||||
test_config,
|
||||
)
|
||||
@@ -1,299 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""EPLB rearrangement consistency tests for quant-method derived state.
|
||||
|
||||
EPLB rearranges every registered Parameter of a RoutedExperts in place,
|
||||
sliced along dim 0 (see RoutedExperts.get_expert_weights). Quant methods
|
||||
must therefore register all derived per-expert tensors as Parameters and
|
||||
alias the same storage in their FusedMoEQuantConfig, so the kernels observe
|
||||
rearranged values with no extra bookkeeping. These tests verify that
|
||||
contract: simulating a rearrangement on the registered Parameters must be
|
||||
indistinguishable from loading a checkpoint with permuted experts.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_w4a4_nvfp4 import ( # noqa: E501
|
||||
CompressedTensorsW4A4Nvfp4MoEMethod,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
EPLB_NVFP4_BACKENDS = ["flashinfer_cutedsl", "flashinfer_trtllm"]
|
||||
|
||||
NUM_EXPERTS = 8
|
||||
HIDDEN_SIZE = 128
|
||||
INTERMEDIATE_SIZE = 256
|
||||
EXPERT_PERMUTATION = [3, 0, 5, 1, 7, 2, 6, 4]
|
||||
|
||||
QUANT_CONFIG_TENSORS = (
|
||||
"w1_scale",
|
||||
"w2_scale",
|
||||
"g1_alphas",
|
||||
"g2_alphas",
|
||||
"a1_gscale",
|
||||
"a2_gscale",
|
||||
)
|
||||
|
||||
|
||||
class _RoutedExpertsStub(torch.nn.Module):
|
||||
"""Minimal ``RoutedExperts`` stand-in for NVFP4 weight processing."""
|
||||
|
||||
def __init__(self, moe_config: FusedMoEConfig) -> None:
|
||||
super().__init__()
|
||||
self.moe_config = moe_config
|
||||
self.activation = moe_config.activation
|
||||
|
||||
def _expert_routing_tables(
|
||||
self,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _make_moe_config(backend: str) -> FusedMoEConfig:
|
||||
parallel_config = FusedMoEParallelConfig(
|
||||
tp_size=1,
|
||||
pcp_size=1,
|
||||
dp_size=1,
|
||||
ep_size=1,
|
||||
tp_rank=0,
|
||||
pcp_rank=0,
|
||||
dp_rank=0,
|
||||
ep_rank=0,
|
||||
sp_size=1,
|
||||
use_ep=True,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
enable_eplb=True,
|
||||
)
|
||||
return FusedMoEConfig(
|
||||
num_experts=NUM_EXPERTS,
|
||||
experts_per_token=2,
|
||||
hidden_dim=HIDDEN_SIZE,
|
||||
intermediate_size=INTERMEDIATE_SIZE,
|
||||
num_local_experts=NUM_EXPERTS,
|
||||
num_logical_experts=NUM_EXPERTS,
|
||||
activation=MoEActivation.SILU,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
moe_parallel_config=parallel_config,
|
||||
in_dtype=torch.bfloat16,
|
||||
moe_backend=backend,
|
||||
)
|
||||
|
||||
|
||||
def _make_raw_weights(
|
||||
device: torch.device, generator: torch.Generator
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Random raw NVFP4 checkpoint tensors, indexable per-expert on dim 0."""
|
||||
e, h, i = NUM_EXPERTS, HIDDEN_SIZE, INTERMEDIATE_SIZE
|
||||
|
||||
def packed(*shape: int) -> torch.Tensor:
|
||||
return torch.randint(
|
||||
0, 256, shape, dtype=torch.uint8, device=device, generator=generator
|
||||
)
|
||||
|
||||
def block_scale(*shape: int) -> torch.Tensor:
|
||||
return torch.randint(
|
||||
0, 128, shape, dtype=torch.uint8, device=device, generator=generator
|
||||
).view(torch.float8_e4m3fn)
|
||||
|
||||
def global_scale(*shape: int) -> torch.Tensor:
|
||||
return torch.rand(shape, device=device, generator=generator) + 0.5
|
||||
|
||||
return {
|
||||
"w13_weight_packed": packed(e, 2 * i, h // 2),
|
||||
"w2_weight_packed": packed(e, h, i // 2),
|
||||
"w13_weight_scale": block_scale(e, 2 * i, h // 16),
|
||||
"w2_weight_scale": block_scale(e, h, i // 16),
|
||||
"w13_weight_global_scale": global_scale(e, 2),
|
||||
"w2_weight_global_scale": global_scale(e),
|
||||
"w13_input_global_scale": global_scale(e, 2),
|
||||
"w2_input_global_scale": global_scale(e),
|
||||
}
|
||||
|
||||
|
||||
def _build_processed_layer(
|
||||
backend: str, raw: dict[str, torch.Tensor], device: torch.device
|
||||
) -> tuple[CompressedTensorsW4A4Nvfp4MoEMethod, _RoutedExpertsStub]:
|
||||
"""Create the method + layer, load ``raw`` and run the real
|
||||
process_weights_after_loading (which builds the flashinfer kernel)."""
|
||||
moe_config = _make_moe_config(backend)
|
||||
method = CompressedTensorsW4A4Nvfp4MoEMethod(moe_config, "layer.0", use_a16=False)
|
||||
layer = _RoutedExpertsStub(moe_config).to(device)
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=NUM_EXPERTS,
|
||||
hidden_size=HIDDEN_SIZE,
|
||||
intermediate_size_per_partition=INTERMEDIATE_SIZE,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=lambda *args, **kwargs: None,
|
||||
)
|
||||
layer = layer.to(device)
|
||||
with torch.no_grad():
|
||||
for name, value in raw.items():
|
||||
getattr(layer, name).copy_(value)
|
||||
method.process_weights_after_loading(layer)
|
||||
return method, layer
|
||||
|
||||
|
||||
def _assert_tensors_equal(
|
||||
actual: torch.Tensor, expected: torch.Tensor, name: str
|
||||
) -> None:
|
||||
if actual.dtype == torch.float8_e4m3fn:
|
||||
assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8)), name
|
||||
elif actual.dtype == torch.uint8:
|
||||
assert torch.equal(actual, expected), name
|
||||
else:
|
||||
torch.testing.assert_close(actual, expected, msg=lambda m: f"{name}: {m}")
|
||||
|
||||
|
||||
def _simulate_eplb_rearrangement(layer: torch.nn.Module, perm: torch.Tensor) -> None:
|
||||
"""Permute experts of every registered Parameter in place, the way
|
||||
EPLB's in-place rearrangement moves expert slices along dim 0."""
|
||||
with torch.no_grad():
|
||||
for _, param in layer.named_parameters():
|
||||
param.copy_(param[perm])
|
||||
|
||||
|
||||
def _ensure_world1_distributed() -> None:
|
||||
"""The enable_eplb config makes weight processing all-reduce activation
|
||||
scale amaxes over the EP group, so a (single-rank) EP group must exist."""
|
||||
from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
)
|
||||
|
||||
if not torch.distributed.is_initialized():
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method="tcp://127.0.0.1:0",
|
||||
local_rank=0,
|
||||
)
|
||||
ensure_model_parallel_initialized(1, 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", EPLB_NVFP4_BACKENDS)
|
||||
def test_nvfp4_eplb_rearrangement_matches_reload(backend: str) -> None:
|
||||
if not (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip("NVFP4 CuteDSL/TRTLLM MoE backends require Blackwell (SM100).")
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
torch.accelerator.set_device_index(device)
|
||||
perm = torch.tensor(EXPERT_PERMUTATION, device=device)
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
_ensure_world1_distributed()
|
||||
generator = torch.Generator(device=device).manual_seed(1234)
|
||||
raw = _make_raw_weights(device, generator)
|
||||
raw_permuted = {name: value[perm].contiguous() for name, value in raw.items()}
|
||||
|
||||
try:
|
||||
method, layer = _build_processed_layer(backend, raw, device)
|
||||
except ValueError as exc:
|
||||
pytest.skip(f"{backend} NVFP4 MoE backend unavailable: {exc}")
|
||||
ref_method, ref_layer = _build_processed_layer(backend, raw_permuted, device)
|
||||
|
||||
# The EPLB contract: every registered Parameter must be an
|
||||
# expert-major contiguous tensor so get_expert_weights can view it
|
||||
# as (E, -1) and rearrange expert slices in place.
|
||||
for name, param in layer.named_parameters():
|
||||
assert param.is_contiguous(), f"{name} is not contiguous"
|
||||
assert param.shape[0] == NUM_EXPERTS, (
|
||||
f"{name} is not expert-major: {tuple(param.shape)}"
|
||||
)
|
||||
|
||||
# Derived per-expert scales must live in registered Parameters that
|
||||
# the quant config aliases; anything else goes stale on rearrangement.
|
||||
quant_config = method.moe_quant_config
|
||||
assert quant_config is not None
|
||||
params = dict(layer.named_parameters())
|
||||
assert quant_config.g1_alphas.data_ptr() == (
|
||||
params["w13_weight_scale_2"].data_ptr()
|
||||
)
|
||||
assert quant_config.g2_alphas.data_ptr() == (
|
||||
params["w2_weight_scale_2"].data_ptr()
|
||||
)
|
||||
assert quant_config.w1_scale.data_ptr() == (
|
||||
params["w13_weight_scale"].data_ptr()
|
||||
)
|
||||
assert quant_config.w2_scale.data_ptr() == params["w2_weight_scale"].data_ptr()
|
||||
|
||||
_simulate_eplb_rearrangement(layer, perm)
|
||||
|
||||
# After rearrangement, all registered state must equal what a fresh
|
||||
# load of the permuted experts produces.
|
||||
ref_params = dict(ref_layer.named_parameters())
|
||||
assert params.keys() == ref_params.keys()
|
||||
for name in params:
|
||||
_assert_tensors_equal(params[name], ref_params[name], name)
|
||||
|
||||
# And so must the kernel-visible quant config tensors (the CuteDSL
|
||||
# MMA scale views alias the registered Parameters' storage).
|
||||
ref_quant_config = ref_method.moe_quant_config
|
||||
assert ref_quant_config is not None
|
||||
for name in QUANT_CONFIG_TENSORS:
|
||||
actual = getattr(quant_config, name)
|
||||
expected = getattr(ref_quant_config, name)
|
||||
assert (actual is None) == (expected is None), name
|
||||
if actual is not None:
|
||||
_assert_tensors_equal(actual, expected, f"quant_config.{name}")
|
||||
|
||||
|
||||
def test_fp8_per_tensor_alphas_registered_and_aliased() -> None:
|
||||
"""CPU-only: the fp8 per-tensor oracle must register the fused
|
||||
(w_scale * a_scale) products as layer Parameters aliased by the quant
|
||||
config, so EPLB rearrangement keeps kernels consistent."""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
make_fp8_moe_quant_config,
|
||||
)
|
||||
|
||||
e = NUM_EXPERTS
|
||||
generator = torch.Generator().manual_seed(1234)
|
||||
|
||||
def scales() -> tuple[torch.Tensor, torch.Tensor]:
|
||||
w_scale = torch.rand((e,), generator=generator) + 0.5
|
||||
a_scale = torch.rand((), generator=generator) + 0.5
|
||||
return w_scale, a_scale.expand(e).contiguous()
|
||||
|
||||
w1_scale, a1_scale = scales()
|
||||
w2_scale, a2_scale = scales()
|
||||
|
||||
layer = torch.nn.Module()
|
||||
quant_config = make_fp8_moe_quant_config(
|
||||
fp8_backend=Fp8MoeBackend.FLASHINFER_CUTLASS,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
params = dict(layer.named_parameters())
|
||||
assert quant_config.g1_alphas.data_ptr() == params["g1_alphas"].data_ptr()
|
||||
assert quant_config.g2_alphas.data_ptr() == params["g2_alphas"].data_ptr()
|
||||
|
||||
perm = torch.tensor(EXPERT_PERMUTATION)
|
||||
ref_config = make_fp8_moe_quant_config(
|
||||
fp8_backend=Fp8MoeBackend.FLASHINFER_CUTLASS,
|
||||
w1_scale=w1_scale[perm].contiguous(),
|
||||
w2_scale=w2_scale[perm].contiguous(),
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
layer=None,
|
||||
)
|
||||
_simulate_eplb_rearrangement(layer, perm)
|
||||
|
||||
torch.testing.assert_close(quant_config.g1_alphas, ref_config.g1_alphas)
|
||||
torch.testing.assert_close(quant_config.g2_alphas, ref_config.g2_alphas)
|
||||
@@ -1,408 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import threading
|
||||
import time
|
||||
|
||||
import msgspec
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_events import (
|
||||
EventBatch,
|
||||
EventPublisherFactory,
|
||||
NullEventPublisher,
|
||||
)
|
||||
|
||||
DP_RANK = 0
|
||||
|
||||
|
||||
class EventSample(
|
||||
msgspec.Struct,
|
||||
tag=True, # type: ignore
|
||||
array_like=True, # type: ignore
|
||||
):
|
||||
"""Test event for publisher testing."""
|
||||
|
||||
id: int
|
||||
value: str
|
||||
|
||||
|
||||
class SampleBatch(EventBatch):
|
||||
"""Test event batch for publisher testing."""
|
||||
|
||||
events: list[EventSample]
|
||||
|
||||
|
||||
def create_test_events(count: int) -> SampleBatch:
|
||||
"""Create a batch of test events."""
|
||||
events = [EventSample(id=i, value=f"test-{i}") for i in range(count)]
|
||||
return SampleBatch(ts=time.time(), events=events)
|
||||
|
||||
|
||||
def test_basic_publishing(publisher, subscriber):
|
||||
"""Test basic event publishing works."""
|
||||
test_batch = create_test_events(5)
|
||||
publisher.publish(test_batch)
|
||||
|
||||
result = subscriber.receive_one(timeout=1000)
|
||||
assert result is not None, "No message received"
|
||||
|
||||
seq, received = result
|
||||
assert seq == 0, "Sequence number mismatch"
|
||||
assert received.ts == pytest.approx(test_batch.ts, abs=0.1), "Timestamp mismatch"
|
||||
assert len(received.events) == len(test_batch.events), "Number of events mismatch"
|
||||
|
||||
for i, event in enumerate(received.events):
|
||||
assert event.id == i, "Event id mismatch"
|
||||
assert event.value == f"test-{i}", "Event value mismatch"
|
||||
|
||||
|
||||
def test_multiple_events(publisher, subscriber):
|
||||
"""Test publishing and receiving multiple event batches."""
|
||||
for _ in range(10):
|
||||
batch = create_test_events(2)
|
||||
publisher.publish(batch)
|
||||
|
||||
received = []
|
||||
for _ in range(10):
|
||||
data = subscriber.receive_one(timeout=100)
|
||||
if data:
|
||||
received.append(data)
|
||||
|
||||
assert len(received) == 10, "Number of messages mismatch"
|
||||
seqs = [seq for seq, _ in received]
|
||||
assert seqs == list(range(10)), "Sequence numbers mismatch"
|
||||
|
||||
|
||||
def test_replay_mechanism(publisher, subscriber):
|
||||
"""Test the replay mechanism works correctly."""
|
||||
for _ in range(19):
|
||||
batch = create_test_events(1)
|
||||
publisher.publish(batch)
|
||||
|
||||
# Drain live events to ensure publisher has buffered them.
|
||||
for _ in range(19):
|
||||
assert subscriber.receive_one(timeout=1000) is not None
|
||||
|
||||
subscriber.request_replay(10)
|
||||
|
||||
replayed = subscriber.receive_replay()
|
||||
|
||||
assert len(replayed) == 9, (
|
||||
f"Expected 9 replayed messages (seq 10-18), got {len(replayed)}"
|
||||
)
|
||||
seqs = [seq for seq, _ in replayed]
|
||||
assert seqs == list(range(10, 19)), "Replayed sequences should be 10-18"
|
||||
|
||||
|
||||
def test_replay_includes_topic(publisher, subscriber, publisher_config):
|
||||
"""Test that replay responses include the topic, matching PUB format."""
|
||||
for _ in range(5):
|
||||
publisher.publish(create_test_events(1))
|
||||
|
||||
# Drain live events to ensure publisher has processed them.
|
||||
for _ in range(5):
|
||||
assert subscriber.receive_one(timeout=1000) is not None
|
||||
|
||||
subscriber.request_replay(0)
|
||||
|
||||
# receive_replay unpacks (topic, seq, payload) and asserts
|
||||
# topic == publisher topic for each message.
|
||||
replayed = subscriber.receive_replay()
|
||||
assert len(replayed) == 5, f"Expected 5 replayed messages, got {len(replayed)}"
|
||||
seqs = [seq for seq, _ in replayed]
|
||||
assert seqs == list(range(5)), "Replayed sequences should be 0-4"
|
||||
|
||||
|
||||
def test_buffer_limit(publisher, subscriber, publisher_config):
|
||||
"""Test buffer limit behavior."""
|
||||
buffer_size = publisher_config.buffer_steps
|
||||
|
||||
# Publish more events than the buffer can hold
|
||||
for i in range(buffer_size + 10):
|
||||
batch = create_test_events(1)
|
||||
publisher.publish(batch)
|
||||
|
||||
time.sleep(0.5) # Need publisher to process above requests
|
||||
subscriber.request_replay(0)
|
||||
|
||||
replayed = subscriber.receive_replay()
|
||||
|
||||
assert len(replayed) == buffer_size, (
|
||||
f"Expected {buffer_size} replayed messages, got {len(replayed)}"
|
||||
)
|
||||
|
||||
seqs = [seq for seq, _ in replayed]
|
||||
assert seqs == list(range(10, buffer_size + 10)), (
|
||||
"Should replay seq 11 through buffer_size+10"
|
||||
)
|
||||
|
||||
|
||||
def test_topic_filtering(publisher_config):
|
||||
"""Test that a subscriber only receives messages matching its topic filter."""
|
||||
publisher_config.replay_endpoint = None
|
||||
|
||||
publisher_config.topic = "foo"
|
||||
pub = EventPublisherFactory.create(publisher_config, DP_RANK)
|
||||
|
||||
from .conftest import MockSubscriber
|
||||
|
||||
sub_foo = MockSubscriber(publisher_config.endpoint, None, "foo")
|
||||
sub_bar = MockSubscriber(publisher_config.endpoint, None, "bar")
|
||||
|
||||
try:
|
||||
time.sleep(0.1)
|
||||
|
||||
for _ in range(3):
|
||||
pub.publish(create_test_events(1))
|
||||
|
||||
foo_received = [sub_foo.receive_one(timeout=200) for _ in range(3)]
|
||||
assert all(msg is not None for msg in foo_received), (
|
||||
"Subscriber with matching topic should receive messages"
|
||||
)
|
||||
|
||||
bar_received = [sub_bar.receive_one(timeout=200) for _ in range(3)]
|
||||
assert all(msg is None for msg in bar_received), (
|
||||
"Subscriber with non-matching topic should receive no messages"
|
||||
)
|
||||
finally:
|
||||
pub.shutdown()
|
||||
sub_foo.close()
|
||||
sub_bar.close()
|
||||
|
||||
|
||||
def test_high_volume(publisher, subscriber):
|
||||
"""Test publishing and receiving a high volume of events."""
|
||||
num_batches = 10_000
|
||||
events_per_batch = 100
|
||||
|
||||
# Publish events in a separate thread to not block
|
||||
def publish_events():
|
||||
for i in range(num_batches):
|
||||
batch = create_test_events(events_per_batch)
|
||||
publisher.publish(batch)
|
||||
# Small delay to avoid overwhelming
|
||||
if i % 100 == 0:
|
||||
time.sleep(0.01)
|
||||
|
||||
received: list[tuple[int, SampleBatch]] = []
|
||||
|
||||
publisher_thread = threading.Thread(target=publish_events)
|
||||
publisher_thread.start()
|
||||
|
||||
start_time = time.time()
|
||||
while len(received) < num_batches:
|
||||
if time.time() - start_time > 10: # Timeout after 10 seconds
|
||||
break
|
||||
|
||||
result = subscriber.receive_one(timeout=100)
|
||||
if result:
|
||||
received.append(result)
|
||||
|
||||
publisher_thread.join()
|
||||
|
||||
assert len(received) >= num_batches * 0.9, "We should have received most messages"
|
||||
|
||||
seqs = [seq for seq, _ in received]
|
||||
assert sorted(seqs) == seqs, "Sequence numbers should be in order"
|
||||
|
||||
|
||||
def test_null_publisher():
|
||||
"""Test that NullEventPublisher can be used without errors."""
|
||||
publisher = NullEventPublisher(DP_RANK)
|
||||
|
||||
# This should not raise any errors
|
||||
batch = create_test_events(5)
|
||||
publisher.publish(batch)
|
||||
publisher.shutdown()
|
||||
|
||||
|
||||
def test_data_parallel_rank_tagging(publisher_config):
|
||||
"""Test that events are properly tagged with their data parallel rank."""
|
||||
publisher_config.topic = "foo"
|
||||
pub_0 = EventPublisherFactory.create(publisher_config, DP_RANK)
|
||||
pub_1 = EventPublisherFactory.create(publisher_config, DP_RANK + 1)
|
||||
|
||||
# Hardcode the expected endpoints based on port offsetting behavior
|
||||
# Both ranks get offsets according to _offset_endpoint_port function
|
||||
base_endpoint = publisher_config.endpoint
|
||||
if "tcp://" in base_endpoint:
|
||||
# For TCP endpoints: tcp://localhost:5557 -> tcp://localhost:5557, tcp://localhost:5558
|
||||
expected_endpoint_0 = base_endpoint # rank 0 gets port + 0 = same port
|
||||
expected_endpoint_1 = base_endpoint.replace(
|
||||
":5557", ":5558"
|
||||
) # rank 1 gets port + 1
|
||||
else:
|
||||
# For inproc endpoints: inproc://test -> inproc://test_dp0, inproc://test_dp1
|
||||
expected_endpoint_0 = base_endpoint # rank 0 gets base
|
||||
expected_endpoint_1 = base_endpoint + "_dp1" # rank 1 gets _dp1
|
||||
|
||||
from .conftest import MockSubscriber
|
||||
|
||||
sub_0 = MockSubscriber(expected_endpoint_0, None, publisher_config.topic)
|
||||
sub_1 = MockSubscriber(expected_endpoint_1, None, publisher_config.topic)
|
||||
|
||||
try:
|
||||
time.sleep(0.1) # Let publishers start up
|
||||
|
||||
# Publish events from different ranks
|
||||
batch_0 = create_test_events(2)
|
||||
batch_1 = create_test_events(3)
|
||||
|
||||
pub_0.publish(batch_0)
|
||||
pub_1.publish(batch_1)
|
||||
|
||||
# Receive events from rank 0
|
||||
result_0 = sub_0.receive_one(timeout=200)
|
||||
assert result_0 is not None, "No message received from rank 0"
|
||||
seq_0, received_0 = result_0
|
||||
|
||||
# Receive events from rank 1
|
||||
result_1 = sub_1.receive_one(timeout=200)
|
||||
assert result_1 is not None, "No message received from rank 1"
|
||||
seq_1, received_1 = result_1
|
||||
|
||||
# Verify DP rank tagging
|
||||
assert received_0.data_parallel_rank == 0, (
|
||||
f"Expected DP rank 0, got {received_0.data_parallel_rank}"
|
||||
)
|
||||
assert received_1.data_parallel_rank == 1, (
|
||||
f"Expected DP rank 1, got {received_1.data_parallel_rank}"
|
||||
)
|
||||
|
||||
# Verify event content is correct
|
||||
assert len(received_0.events) == 2, "Wrong number of events from rank 0"
|
||||
assert len(received_1.events) == 3, "Wrong number of events from rank 1"
|
||||
|
||||
finally:
|
||||
pub_0.shutdown()
|
||||
pub_1.shutdown()
|
||||
sub_0.close()
|
||||
sub_1.close()
|
||||
|
||||
|
||||
def test_event_publisher_factory(random_port):
|
||||
"""Test event publisher factory creation behavior under different configurations."""
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed.kv_events import ZmqEventPublisher
|
||||
|
||||
# test config is None
|
||||
publisher = EventPublisherFactory.create(None, DP_RANK)
|
||||
assert isinstance(publisher, NullEventPublisher)
|
||||
publisher.shutdown()
|
||||
|
||||
# test disable kv cache events
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=False,
|
||||
publisher="zmq", # Even if zmq is specified, should return NullEventPublisher
|
||||
endpoint="tcp://localhost:5557",
|
||||
)
|
||||
publisher = EventPublisherFactory.create(config, DP_RANK)
|
||||
assert isinstance(publisher, NullEventPublisher)
|
||||
publisher.shutdown()
|
||||
|
||||
# test zmq publisher
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
endpoint=f"tcp://*:{random_port}",
|
||||
replay_endpoint=f"tcp://*:{random_port + 100}",
|
||||
)
|
||||
publisher = EventPublisherFactory.create(config, DP_RANK + 1)
|
||||
assert isinstance(publisher, ZmqEventPublisher)
|
||||
resolved_config = publisher.get_publisher_config()
|
||||
assert resolved_config.endpoint == f"tcp://*:{random_port + 1}"
|
||||
assert resolved_config.replay_endpoint == f"tcp://*:{random_port + 101}"
|
||||
assert config.endpoint == f"tcp://*:{random_port}"
|
||||
publisher.shutdown()
|
||||
|
||||
# test unknown publisher
|
||||
with pytest.raises(ValueError, match="Input should be"):
|
||||
KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="unknown_publisher",
|
||||
endpoint="tcp://localhost:5557",
|
||||
)
|
||||
|
||||
# test publisher not specified
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
# publisher not specified, should default to "zmq"
|
||||
endpoint="tcp://localhost:5557",
|
||||
)
|
||||
publisher = EventPublisherFactory.create(config, DP_RANK)
|
||||
assert isinstance(publisher, ZmqEventPublisher)
|
||||
publisher.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("endpoint", "expected"),
|
||||
[("tcp://*:5557", "tcp://*:5558"), ("tcp://*:0", "tcp://*:0")],
|
||||
)
|
||||
def test_offset_endpoint_port_keeps_port_zero(endpoint, expected):
|
||||
"""Port 0 is not offset: every rank asks the OS for its own port."""
|
||||
from vllm.distributed.kv_events import ZmqEventPublisher
|
||||
|
||||
assert ZmqEventPublisher.offset_endpoint_port(endpoint, 1) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("node_ip", ["127.0.0.1", "::1"])
|
||||
def test_ephemeral_publishers_report_bound_endpoints(monkeypatch, node_ip):
|
||||
"""Two ranks binding tcp://*:0 get distinct OS-assigned ports on this
|
||||
node's IP and report endpoints a subscriber can use for events and replay."""
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed import kv_events
|
||||
from vllm.utils.network_utils import split_zmq_path
|
||||
|
||||
from .conftest import MockSubscriber
|
||||
|
||||
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
endpoint="tcp://*:0",
|
||||
replay_endpoint="tcp://*:0",
|
||||
topic="kv",
|
||||
)
|
||||
pubs = [EventPublisherFactory.create(config, rank) for rank in range(2)]
|
||||
try:
|
||||
resolved = [pub.get_publisher_config() for pub in pubs]
|
||||
endpoints = [e for c in resolved for e in (c.endpoint, c.replay_endpoint)]
|
||||
hosts_ports = {split_zmq_path(e)[1:] for e in endpoints}
|
||||
assert {h for h, _ in hosts_ports} == {node_ip}
|
||||
assert len(hosts_ports) == 4 and "0" not in {p for _, p in hosts_ports}
|
||||
assert config.endpoint == "tcp://*:0"
|
||||
|
||||
sub = MockSubscriber(endpoints[::2], endpoints[1::2], topic="kv")
|
||||
try:
|
||||
# PUB/SUB drops messages sent before the subscription lands, so
|
||||
# keep publishing until both ranks are heard.
|
||||
ranks: set[int] = set()
|
||||
deadline = time.time() + 10
|
||||
while ranks != {0, 1} and time.time() < deadline:
|
||||
for pub in pubs:
|
||||
pub.publish(create_test_events(1))
|
||||
while (result := sub.receive_one(timeout=100)) is not None:
|
||||
ranks.add(result[1].data_parallel_rank)
|
||||
assert ranks == {0, 1}
|
||||
sub.request_replay(0, socket_idx=1)
|
||||
assert sub.receive_replay(socket_idx=1)
|
||||
finally:
|
||||
sub.close()
|
||||
finally:
|
||||
for pub in pubs:
|
||||
pub.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("node_ip", ["0.0.0.0", "::"])
|
||||
def test_ephemeral_bind_needs_a_node_ip(monkeypatch, node_ip):
|
||||
"""tcp://*:0 is advertised as this node's IP; with no usable IP the
|
||||
publisher refuses instead of reporting a wildcard."""
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed import kv_events
|
||||
|
||||
monkeypatch.setattr(kv_events, "get_ip", lambda: node_ip)
|
||||
config = KVEventsConfig(
|
||||
enable_kv_cache_events=True, publisher="zmq", endpoint="tcp://*:0"
|
||||
)
|
||||
with pytest.raises(ValueError, match="VLLM_HOST_IP"):
|
||||
EventPublisherFactory.create(config, DP_RANK)
|
||||
@@ -1,146 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.device_communicators import cuda_communicator
|
||||
from vllm.distributed.device_communicators import (
|
||||
flashinfer_pcie_ipc_all_reduce as pcie_ipc,
|
||||
)
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
|
||||
|
||||
def _uninitialized_comm() -> pcie_ipc.FlashInferPcieIpcAllReduce:
|
||||
comm = pcie_ipc.FlashInferPcieIpcAllReduce.__new__(
|
||||
pcie_ipc.FlashInferPcieIpcAllReduce
|
||||
)
|
||||
comm.disabled = False
|
||||
comm.group = object()
|
||||
comm.tune_group = object()
|
||||
comm.device = torch.device("cuda:0")
|
||||
comm.world_size = 4
|
||||
comm.rank = 0
|
||||
comm.workspace = None
|
||||
comm.hidden_dim = 0
|
||||
comm.dtype = None
|
||||
return comm
|
||||
|
||||
|
||||
def test_setup_uses_exact_graph_capacity(monkeypatch, tmp_path):
|
||||
workspace = Mock()
|
||||
factory = Mock(return_value=workspace)
|
||||
monkeypatch.setattr(
|
||||
pcie_ipc,
|
||||
"flashinfer_comm",
|
||||
SimpleNamespace(PcieIpcAllReduceWorkspace=factory),
|
||||
)
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", Mock())
|
||||
|
||||
comm = _uninitialized_comm()
|
||||
cache = tmp_path / "tp4.json"
|
||||
comm.setup(
|
||||
hidden_dim=4096,
|
||||
dtype=torch.bfloat16,
|
||||
capture_sizes=[32, 8, 16, 32],
|
||||
tune_cache=cache,
|
||||
)
|
||||
|
||||
factory.assert_called_once_with(
|
||||
group=comm.group,
|
||||
max_numel=32 * 4096,
|
||||
dtype=torch.bfloat16,
|
||||
tune_batches=(8, 16, 32),
|
||||
tune_cache=str(cache),
|
||||
)
|
||||
workspace.tune.assert_called_once_with(
|
||||
[4096], dtype=torch.bfloat16, tune_group=comm.tune_group
|
||||
)
|
||||
workspace.prepare.assert_called_once_with(
|
||||
[(8, 4096), (16, 4096), (32, 4096)], dtype=torch.bfloat16
|
||||
)
|
||||
assert workspace.rebind_stream.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("shape", "dtype", "contiguous", "expected"),
|
||||
[
|
||||
((16, 4096), torch.bfloat16, True, True),
|
||||
((16, 2048), torch.bfloat16, True, False),
|
||||
((16, 4096), torch.float16, True, False),
|
||||
((16, 4096), torch.bfloat16, False, False),
|
||||
],
|
||||
)
|
||||
def test_should_use_only_prepared_hidden_shape(shape, dtype, contiguous, expected):
|
||||
comm = _uninitialized_comm()
|
||||
comm.hidden_dim = 4096
|
||||
comm.dtype = torch.bfloat16
|
||||
comm.workspace = Mock()
|
||||
comm.workspace.supports.return_value = True
|
||||
inp = SimpleNamespace(
|
||||
is_cuda=True,
|
||||
is_contiguous=lambda: contiguous,
|
||||
dim=lambda: len(shape),
|
||||
shape=shape,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
assert comm.should_use(inp) is expected
|
||||
|
||||
|
||||
def test_capture_rebinds_workspace_even_on_error(monkeypatch):
|
||||
comm = _uninitialized_comm()
|
||||
comm.workspace = Mock()
|
||||
synchronize = Mock()
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", synchronize)
|
||||
|
||||
with pytest.raises(RuntimeError, match="capture failed"), comm.capture():
|
||||
raise RuntimeError("capture failed")
|
||||
|
||||
assert synchronize.call_count == 2
|
||||
assert comm.workspace.rebind_stream.call_count == 2
|
||||
|
||||
|
||||
def test_cuda_dispatch_prefers_pcie_ipc_before_existing_flashinfer(monkeypatch):
|
||||
communicator = cuda_communicator.CudaCommunicator.__new__(
|
||||
cuda_communicator.CudaCommunicator
|
||||
)
|
||||
communicator.pynccl_comm = Mock(world_size=4)
|
||||
communicator.qr_comm = None
|
||||
communicator.fi_pcie_ipc_ar_comm = Mock()
|
||||
communicator.fi_pcie_ipc_ar_comm.should_use.return_value = True
|
||||
expected = object()
|
||||
communicator.fi_pcie_ipc_ar_comm.all_reduce.return_value = expected
|
||||
communicator.fi_ar_comm = Mock()
|
||||
communicator.aiter_ar_comm = None
|
||||
communicator.ca_comm = None
|
||||
communicator.symm_mem_comm = None
|
||||
monkeypatch.setattr(
|
||||
cuda_communicator, "should_nccl_symm_mem_allreduce", lambda *_: False
|
||||
)
|
||||
|
||||
assert communicator.all_reduce(object()) is expected
|
||||
communicator.fi_ar_comm.should_use_fi_ar.assert_not_called()
|
||||
|
||||
|
||||
def test_group_destroy_releases_communicator_before_process_groups(monkeypatch):
|
||||
events = []
|
||||
coordinator = GroupCoordinator.__new__(GroupCoordinator)
|
||||
coordinator.device_communicator = SimpleNamespace(
|
||||
destroy=lambda: events.append("communicator")
|
||||
)
|
||||
coordinator.device_group = object()
|
||||
coordinator.cpu_group = object()
|
||||
coordinator.mq_broadcaster = None
|
||||
monkeypatch.setattr(
|
||||
torch.distributed,
|
||||
"destroy_process_group",
|
||||
lambda _group: events.append("process_group"),
|
||||
)
|
||||
|
||||
coordinator.destroy()
|
||||
|
||||
assert events == ["communicator", "process_group", "process_group"]
|
||||
@@ -1,238 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import VllmRunner
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
from vllm import SamplingParams, TokensPrompt
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
|
||||
|
||||
MODEL = "moonshotai/Kimi-Linear-48B-A3B-Instruct"
|
||||
|
||||
|
||||
def _make_tiny_overrides() -> dict:
|
||||
linear_attn_config = {
|
||||
"full_attn_layers": [4],
|
||||
"head_dim": 128,
|
||||
"kda_layers": [1, 2, 3],
|
||||
"num_heads": 8,
|
||||
"short_conv_kernel_size": 4,
|
||||
"use_full_rank_gate": True,
|
||||
}
|
||||
|
||||
return {
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
"vocab_size": 256,
|
||||
"hidden_size": 256,
|
||||
"head_dim": 32,
|
||||
"intermediate_size": 512,
|
||||
"num_hidden_layers": 4,
|
||||
"num_attention_heads": 8,
|
||||
"num_key_value_heads": 8,
|
||||
"num_experts": None,
|
||||
"num_experts_per_token": None,
|
||||
"num_shared_experts": 0,
|
||||
"q_lora_rank": 128,
|
||||
"kv_lora_rank": 512,
|
||||
"qk_nope_head_dim": 128,
|
||||
"qk_rope_head_dim": 64,
|
||||
"v_head_dim": 128,
|
||||
"mla_use_nope": True,
|
||||
"mla_use_output_gate": True,
|
||||
"linear_attn_config": linear_attn_config,
|
||||
"max_position_embeddings": 512,
|
||||
"model_max_length": 512,
|
||||
}
|
||||
|
||||
|
||||
def _run_tiny_model(
|
||||
vllm_runner: type[VllmRunner], dcp_size: int
|
||||
) -> list[tuple[list[int], list[float]]]:
|
||||
with vllm_runner(
|
||||
model_name=MODEL,
|
||||
skip_tokenizer_init=True,
|
||||
load_format="dummy",
|
||||
hf_overrides=_make_tiny_overrides(),
|
||||
tensor_parallel_size=2,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
distributed_executor_backend="mp",
|
||||
dtype="bfloat16",
|
||||
seed=0,
|
||||
enforce_eager=True,
|
||||
max_model_len=512,
|
||||
max_num_seqs=8,
|
||||
max_num_batched_tokens=1024,
|
||||
gpu_memory_utilization=0.25,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=False,
|
||||
) as runner:
|
||||
lengths = [1, 32, 33, 129]
|
||||
prompts = [
|
||||
TokensPrompt(
|
||||
prompt_token_ids=[
|
||||
3 + ((request_idx + token_idx) % 251) for token_idx in range(length)
|
||||
]
|
||||
)
|
||||
for request_idx, length in enumerate(lengths)
|
||||
]
|
||||
outputs = runner.llm.generate(
|
||||
prompts,
|
||||
SamplingParams(
|
||||
temperature=0,
|
||||
max_tokens=8,
|
||||
seed=0,
|
||||
ignore_eos=True,
|
||||
logprobs=20,
|
||||
),
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
results = []
|
||||
for request_output in outputs:
|
||||
completion = request_output.outputs[0]
|
||||
token_ids = list(completion.token_ids)
|
||||
assert completion.logprobs is not None
|
||||
selected_logprobs = [
|
||||
step_logprobs[token_id].logprob
|
||||
for token_id, step_logprobs in zip(token_ids, completion.logprobs)
|
||||
]
|
||||
results.append((token_ids, selected_logprobs))
|
||||
return results
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.distributed(num_gpus=2)
|
||||
def test_kimi_linear_dcp_tiny(
|
||||
vllm_runner: type[VllmRunner],
|
||||
num_gpus_available: int,
|
||||
) -> None:
|
||||
if num_gpus_available < 2:
|
||||
pytest.skip("Need at least 2 GPUs")
|
||||
|
||||
baseline = _run_tiny_model(vllm_runner, dcp_size=1)
|
||||
dcp = _run_tiny_model(vllm_runner, dcp_size=2)
|
||||
|
||||
assert [tokens for tokens, _ in dcp] == [tokens for tokens, _ in baseline]
|
||||
logprob_drifts = []
|
||||
for (_, baseline_logprobs), (_, dcp_logprobs) in zip(baseline, dcp):
|
||||
for baseline_logprob, dcp_logprob in zip(baseline_logprobs, dcp_logprobs):
|
||||
assert math.isfinite(baseline_logprob)
|
||||
assert math.isfinite(dcp_logprob)
|
||||
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
|
||||
|
||||
assert max(logprob_drifts) <= 1e-2
|
||||
|
||||
|
||||
def _make_tiny_k3_config(model_dir: Path) -> str:
|
||||
config = KimiK3Config(
|
||||
text_config=_make_tiny_overrides(),
|
||||
vision_config={
|
||||
"vt_num_attention_heads": 2,
|
||||
"vt_num_hidden_layers": 1,
|
||||
"vt_hidden_size": 32,
|
||||
"vt_intermediate_size": 64,
|
||||
"qkv_hidden_size": 48,
|
||||
},
|
||||
architectures=["KimiK3ForConditionalGeneration"],
|
||||
)
|
||||
config.save_pretrained(model_dir)
|
||||
return str(model_dir)
|
||||
|
||||
|
||||
def _run_k3_partial_prefix_reuse(
|
||||
vllm_runner: type[VllmRunner], model_name: str, dcp_size: int
|
||||
) -> tuple[list[int], list[float], int]:
|
||||
block_size = 256
|
||||
common_prefix = [3 + (token_idx % 251) for token_idx in range(block_size)]
|
||||
prime_prompt = TokensPrompt(prompt_token_ids=common_prefix + [17])
|
||||
replay_prompt = TokensPrompt(prompt_token_ids=common_prefix + [18, 19])
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0,
|
||||
max_tokens=8,
|
||||
seed=0,
|
||||
ignore_eos=True,
|
||||
logprobs=20,
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model_name=model_name,
|
||||
skip_tokenizer_init=True,
|
||||
load_format="dummy",
|
||||
language_model_only=True,
|
||||
tensor_parallel_size=2,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
distributed_executor_backend="mp",
|
||||
dtype="bfloat16",
|
||||
seed=0,
|
||||
max_model_len=512,
|
||||
max_num_seqs=4,
|
||||
max_num_batched_tokens=256,
|
||||
gpu_memory_utilization=0.85,
|
||||
block_size=block_size,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=True,
|
||||
) as runner:
|
||||
assert (
|
||||
runner.llm.llm_engine.vllm_config.compilation_config.cudagraph_mode
|
||||
== CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
)
|
||||
runner.llm.generate(
|
||||
[prime_prompt],
|
||||
sampling_params,
|
||||
use_tqdm=False,
|
||||
)
|
||||
replay_output = runner.llm.generate(
|
||||
[replay_prompt],
|
||||
sampling_params,
|
||||
use_tqdm=False,
|
||||
)[0]
|
||||
|
||||
completion = replay_output.outputs[0]
|
||||
token_ids = list(completion.token_ids)
|
||||
assert len(token_ids) == sampling_params.max_tokens
|
||||
assert completion.logprobs is not None
|
||||
assert len(completion.logprobs) == sampling_params.max_tokens
|
||||
selected_logprobs = [
|
||||
step_logprobs[token_id].logprob
|
||||
for token_id, step_logprobs in zip(token_ids, completion.logprobs)
|
||||
]
|
||||
return token_ids, selected_logprobs, replay_output.num_cached_tokens
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.distributed(num_gpus=2)
|
||||
def test_kimi_k3_dcp_partial_prefix_reuse(
|
||||
vllm_runner: type[VllmRunner],
|
||||
num_gpus_available: int,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
if num_gpus_available < 2:
|
||||
pytest.skip("Need at least 2 GPUs")
|
||||
|
||||
model_name = _make_tiny_k3_config(tmp_path / "tiny-kimi-k3")
|
||||
baseline_tokens, baseline_logprobs, baseline_cached = _run_k3_partial_prefix_reuse(
|
||||
vllm_runner, model_name, dcp_size=1
|
||||
)
|
||||
dcp_tokens, dcp_logprobs, dcp_cached = _run_k3_partial_prefix_reuse(
|
||||
vllm_runner, model_name, dcp_size=2
|
||||
)
|
||||
|
||||
assert baseline_cached == 256
|
||||
assert dcp_cached == 256
|
||||
assert dcp_tokens == baseline_tokens
|
||||
assert len(dcp_logprobs) == len(baseline_logprobs) == 8
|
||||
logprob_drifts = []
|
||||
for baseline_logprob, dcp_logprob in zip(baseline_logprobs, dcp_logprobs):
|
||||
assert math.isfinite(baseline_logprob)
|
||||
assert math.isfinite(dcp_logprob)
|
||||
logprob_drifts.append(abs(baseline_logprob - dcp_logprob))
|
||||
assert max(logprob_drifts) <= 1e-2
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,498 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Integration tests for MultiprocExecutor at the executor level.
|
||||
This test directly tests the executor without going through the LLM interface,
|
||||
focusing on executor initialization, RPC calls, and distributed execution.
|
||||
"""
|
||||
|
||||
import multiprocessing
|
||||
import os
|
||||
import socket
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.executor import multiproc_executor
|
||||
from vllm.v1.executor.multiproc_executor import MultiprocExecutor
|
||||
|
||||
MODEL = "facebook/opt-125m"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_world_size", "data_parallel_size_local", "expected_num_local_procs"),
|
||||
[(1, 4, 4), (4, 1, 4), (2, 0, 2)],
|
||||
)
|
||||
def test_multiproc_executor_counts_all_local_dp_workers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
local_world_size: int,
|
||||
data_parallel_size_local: int,
|
||||
expected_num_local_procs: int,
|
||||
):
|
||||
"""All colocated DP workers share the node's startup CPU budget."""
|
||||
executor = object.__new__(MultiprocExecutor)
|
||||
executor.world_size = local_world_size
|
||||
executor.local_world_size = local_world_size
|
||||
executor.parallel_config = type(
|
||||
"ParallelConfig",
|
||||
(),
|
||||
{"data_parallel_size_local": data_parallel_size_local},
|
||||
)()
|
||||
|
||||
monkeypatch.setattr(
|
||||
executor,
|
||||
"_get_parallel_sizes",
|
||||
lambda: (local_world_size, 1, 1),
|
||||
)
|
||||
|
||||
class StopExecutorInit(Exception):
|
||||
pass
|
||||
|
||||
def capture_num_local_procs(num_local_procs: int):
|
||||
assert num_local_procs == expected_num_local_procs
|
||||
raise StopExecutorInit
|
||||
|
||||
monkeypatch.setattr(
|
||||
multiproc_executor,
|
||||
"set_multiprocessing_worker_envs",
|
||||
capture_num_local_procs,
|
||||
)
|
||||
|
||||
with pytest.raises(StopExecutorInit):
|
||||
executor._init_executor()
|
||||
executor._finalizer.detach()
|
||||
|
||||
|
||||
def create_vllm_config(
|
||||
tensor_parallel_size: int = 1,
|
||||
pipeline_parallel_size: int = 1,
|
||||
max_model_len: int = 256,
|
||||
gpu_memory_utilization: float = 0.3,
|
||||
distributed_executor_backend: str = "mp",
|
||||
nnodes: int = 1,
|
||||
node_rank: int = 0,
|
||||
master_port: int = 0,
|
||||
) -> VllmConfig:
|
||||
"""Create a VllmConfig for testing using EngineArgs."""
|
||||
engine_args = EngineArgs(
|
||||
model=MODEL,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
pipeline_parallel_size=pipeline_parallel_size,
|
||||
max_model_len=max_model_len,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enforce_eager=True,
|
||||
)
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
|
||||
# Override distributed node settings if needed
|
||||
if nnodes > 1 or node_rank > 0:
|
||||
vllm_config.parallel_config.nnodes = nnodes
|
||||
vllm_config.parallel_config.node_rank = node_rank
|
||||
vllm_config.parallel_config.master_port = master_port
|
||||
if nnodes > 1:
|
||||
vllm_config.parallel_config.disable_custom_all_reduce = True
|
||||
|
||||
return vllm_config
|
||||
|
||||
|
||||
def create_test_scheduler_output(num_requests: int = 1) -> SchedulerOutput:
|
||||
"""Create a minimal SchedulerOutput for testing."""
|
||||
# This is a simplified version - in practice you'd need proper
|
||||
# SchedulerOutput construction based on the actual vLLM v1 API
|
||||
return SchedulerOutput(
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_resumed_reqs=[],
|
||||
scheduled_running_reqs=[],
|
||||
num_scheduled_tokens={},
|
||||
total_num_scheduled_tokens=0,
|
||||
)
|
||||
|
||||
|
||||
def test_multiproc_executor_initialization():
|
||||
"""Test that MultiprocExecutor can be initialized with proper config."""
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
)
|
||||
|
||||
# Create executor - this should initialize workers
|
||||
executor = MultiprocExecutor(vllm_config=vllm_config)
|
||||
|
||||
# Verify executor properties
|
||||
assert executor.world_size == 1, "World size should be 1 for single GPU"
|
||||
assert executor.local_world_size == 1, "Local world size should be 1"
|
||||
assert hasattr(executor, "workers"), "Executor should have workers"
|
||||
assert len(executor.workers) == 1, "Should have 1 worker for single GPU"
|
||||
|
||||
# Clean up
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_multiproc_executor_initialization_tensor_parallel():
|
||||
"""Test MultiprocExecutor initialization with tensor parallelism."""
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=2,
|
||||
pipeline_parallel_size=1,
|
||||
)
|
||||
|
||||
# Create executor
|
||||
executor = MultiprocExecutor(vllm_config=vllm_config)
|
||||
|
||||
# Verify executor properties
|
||||
assert executor.world_size == 2, "World size should be 2 for TP=2"
|
||||
assert executor.local_world_size == 2, "Local world size should be 2"
|
||||
assert len(executor.workers) == 2, "Should have 2 workers for TP=2"
|
||||
|
||||
# Verify output rank calculation
|
||||
output_rank = executor._get_output_rank()
|
||||
assert output_rank == 0, "Output rank should be 0 for TP=2, PP=1"
|
||||
|
||||
# Clean up
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_multiproc_executor_collective_rpc():
|
||||
"""Test collective RPC calls to all workers."""
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=2,
|
||||
pipeline_parallel_size=1,
|
||||
)
|
||||
|
||||
# Create executor
|
||||
executor = MultiprocExecutor(vllm_config=vllm_config)
|
||||
|
||||
try:
|
||||
# Test check_health RPC - should work without errors
|
||||
executor.check_health()
|
||||
|
||||
# Test that RPC works correctly
|
||||
# Note: We're just testing that the RPC mechanism works,
|
||||
# not testing actual model execution here
|
||||
assert not executor.is_failed, "Executor should not be in failed state"
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
def test_multiproc_executor_failure_callback():
|
||||
"""Test failure callback registration and invocation."""
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
)
|
||||
|
||||
executor = MultiprocExecutor(vllm_config=vllm_config)
|
||||
|
||||
try:
|
||||
# Test callback registration
|
||||
callback_invoked = []
|
||||
|
||||
def test_callback():
|
||||
callback_invoked.append(True)
|
||||
|
||||
# Register callback
|
||||
executor.register_failure_callback(test_callback)
|
||||
|
||||
# Callback should not be invoked yet
|
||||
assert len(callback_invoked) == 0, "Callback should not be invoked immediately"
|
||||
|
||||
# Simulate failure
|
||||
executor.is_failed = True
|
||||
|
||||
# Register another callback - should be invoked immediately
|
||||
executor.register_failure_callback(test_callback)
|
||||
assert len(callback_invoked) == 1, (
|
||||
"Callback should be invoked when executor is failed"
|
||||
)
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_multiproc_executor_worker_monitor():
|
||||
"""Test that worker monitor is set up correctly."""
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=2,
|
||||
pipeline_parallel_size=1,
|
||||
)
|
||||
|
||||
executor = MultiprocExecutor(vllm_config=vllm_config)
|
||||
|
||||
try:
|
||||
# Verify all worker processes are alive
|
||||
for worker in executor.workers:
|
||||
assert worker.proc.is_alive(), f"Worker rank {worker.rank} should be alive"
|
||||
|
||||
# Verify executor is not in failed state
|
||||
assert not executor.is_failed, "Executor should not be in failed state"
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
executor.shutdown()
|
||||
|
||||
# After shutdown, workers should be terminated
|
||||
import time
|
||||
|
||||
time.sleep(0.5) # Give processes time to terminate
|
||||
for worker in executor.workers:
|
||||
assert not worker.proc.is_alive(), (
|
||||
f"Worker rank {worker.rank} should terminate after shutdown"
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_multiproc_executor_get_response_message_queues():
|
||||
"""Test message queue retrieval for different ranks."""
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=2,
|
||||
pipeline_parallel_size=1,
|
||||
)
|
||||
|
||||
executor = MultiprocExecutor(vllm_config=vllm_config)
|
||||
|
||||
try:
|
||||
# Get all message queues
|
||||
all_queues = executor.get_response_mqs()
|
||||
assert len(all_queues) == 2, "Should have 2 message queues for 2 workers"
|
||||
|
||||
# Get message queue for specific rank
|
||||
rank0_queue = executor.get_response_mqs(unique_reply_rank=0)
|
||||
assert len(rank0_queue) == 1, "Should have 1 message queue for rank 0"
|
||||
|
||||
rank1_queue = executor.get_response_mqs(unique_reply_rank=1)
|
||||
assert len(rank1_queue) == 1, "Should have 1 message queue for rank 1"
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
def test_multiproc_executor_shutdown_cleanup():
|
||||
"""Test that shutdown properly cleans up resources."""
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
)
|
||||
|
||||
executor = MultiprocExecutor(vllm_config=vllm_config)
|
||||
|
||||
# Verify executor is set up
|
||||
assert hasattr(executor, "workers"), "Executor should have workers"
|
||||
assert len(executor.workers) > 0, "Should have at least one worker"
|
||||
|
||||
# Shutdown
|
||||
executor.shutdown()
|
||||
|
||||
# Verify cleanup
|
||||
import time
|
||||
|
||||
time.sleep(0.5) # Give processes time to terminate
|
||||
|
||||
for worker in executor.workers:
|
||||
assert not worker.proc.is_alive(), "Worker processes should be terminated"
|
||||
|
||||
# Verify shutdown flag is set
|
||||
assert executor.shutting_down, "Shutdown flag should be set"
|
||||
|
||||
# Multiple shutdowns should be safe (idempotent)
|
||||
executor.shutdown()
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_multiproc_executor_pipeline_parallel():
|
||||
"""Test MultiprocExecutor with pipeline parallelism."""
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=2,
|
||||
pipeline_parallel_size=2,
|
||||
)
|
||||
|
||||
executor = MultiprocExecutor(vllm_config=vllm_config)
|
||||
|
||||
try:
|
||||
# Verify executor properties
|
||||
assert executor.world_size == 4, "World size should be 4 for TP=2, PP=2"
|
||||
assert len(executor.workers) == 4, "Should have 4 workers"
|
||||
|
||||
# Verify output rank calculation
|
||||
# For TP=2, PP=2: output should be from the last PP stage (ranks 2-3)
|
||||
# Specifically rank 2 (first rank of last PP stage)
|
||||
output_rank = executor._get_output_rank()
|
||||
assert output_rank == 2, "Output rank should be 2 (first rank of last PP stage)"
|
||||
|
||||
# V2 model runner uses one extra batch to overlap async scheduling.
|
||||
expected_concurrent_batches = 2 + int(
|
||||
vllm_config.scheduler_config.async_scheduling
|
||||
and vllm_config.use_v2_model_runner
|
||||
)
|
||||
assert vllm_config.max_concurrent_batches == expected_concurrent_batches, (
|
||||
"Max concurrent batches should follow the configured PP/async "
|
||||
"scheduling policy"
|
||||
)
|
||||
finally:
|
||||
# Clean up
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
def test_multiproc_executor_properties():
|
||||
"""Test various executor properties and configurations."""
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
)
|
||||
|
||||
executor = MultiprocExecutor(vllm_config=vllm_config)
|
||||
|
||||
try:
|
||||
# Test supports_pp property
|
||||
assert MultiprocExecutor.supports_pp is True, (
|
||||
"MultiprocExecutor should support pipeline parallelism"
|
||||
)
|
||||
|
||||
# Test world_size calculation
|
||||
assert executor.world_size == (
|
||||
executor.parallel_config.tensor_parallel_size
|
||||
* executor.parallel_config.pipeline_parallel_size
|
||||
), "World size should equal TP * PP"
|
||||
|
||||
# Test local_world_size calculation
|
||||
assert executor.local_world_size == (
|
||||
executor.parallel_config.world_size // executor.parallel_config.nnodes
|
||||
), "Local world size should be world_size / nnodes"
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_multiproc_executor_multi_node():
|
||||
"""Test MultiprocExecutor with multi-node configuration.
|
||||
This simulates 2 nodes with TP=4:
|
||||
- Node 0 (rank 0): Uses GPUs 0,1 (device_control_env_var=0,1) with TP=2
|
||||
- Node 1 (rank 1): Uses GPUs 2,3 (device_control_env_var=2,3) with TP=2
|
||||
Total world_size = 4, nnodes = 2
|
||||
"""
|
||||
# Python 3.14+ changed default multiprocessing start method to 'forkserver'
|
||||
# which cannot pickle nested functions. Use 'fork' for this test.
|
||||
mp_ctx = multiprocessing.get_context("fork")
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("", 0))
|
||||
port = s.getsockname()[1]
|
||||
# symm_mem does not work for simulating multi instance in single node
|
||||
os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0"
|
||||
|
||||
# Use the platform-specific device control env var (e.g.
|
||||
# CUDA_VISIBLE_DEVICES on CUDA, ZE_AFFINITY_MASK on XPU) so that each
|
||||
# node only sees its own devices on different platform.
|
||||
device_control_env_var = current_platform.device_control_env_var
|
||||
|
||||
def run_node(node_rank: int, result_queue: multiprocessing.Queue, port: int):
|
||||
"""Run a single node's executor."""
|
||||
executor = None
|
||||
try:
|
||||
# Restrict visible devices for this node
|
||||
if node_rank == 0:
|
||||
os.environ[device_control_env_var] = "0,1"
|
||||
else:
|
||||
os.environ[device_control_env_var] = "2,3"
|
||||
|
||||
# Create config for this node
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=4, # Total TP across all nodes
|
||||
pipeline_parallel_size=1,
|
||||
nnodes=2, # 2 nodes
|
||||
node_rank=node_rank,
|
||||
master_port=port, # same port
|
||||
)
|
||||
|
||||
# Create executor for this node
|
||||
executor = MultiprocExecutor(vllm_config=vllm_config)
|
||||
|
||||
# Verify node-specific properties
|
||||
assert executor.world_size == 4, (
|
||||
f"World size should be 4 on node {node_rank}"
|
||||
)
|
||||
assert executor.local_world_size == 2, (
|
||||
f"Local world size should be 2 on node {node_rank}"
|
||||
)
|
||||
assert len(executor.workers) == 2, (
|
||||
f"Should have 2 local workers on node {node_rank}"
|
||||
)
|
||||
|
||||
# Verify worker ranks are correct for this node
|
||||
expected_ranks = [node_rank * 2, node_rank * 2 + 1]
|
||||
actual_ranks = sorted([w.rank for w in executor.workers])
|
||||
assert actual_ranks == expected_ranks, (
|
||||
f"Node {node_rank} should have workers "
|
||||
f"with ranks {expected_ranks}, got {actual_ranks}"
|
||||
)
|
||||
# Verify all workers are alive
|
||||
for worker in executor.workers:
|
||||
assert worker.proc.is_alive(), (
|
||||
f"Worker rank {worker.rank} should be alive on node {node_rank}"
|
||||
)
|
||||
# executor.gen
|
||||
# Put success result in queue BEFORE shutdown to avoid hanging
|
||||
result_queue.put({"node": node_rank, "success": True})
|
||||
import time
|
||||
|
||||
time.sleep(2)
|
||||
executor.shutdown()
|
||||
except Exception as e:
|
||||
# Put failure result in queue
|
||||
result_queue.put({"node": node_rank, "success": False, "error": str(e)})
|
||||
raise e
|
||||
finally:
|
||||
if executor is not None:
|
||||
executor.shutdown()
|
||||
|
||||
# Create a queue to collect results from both processes
|
||||
result_queue: multiprocessing.Queue[dict[str, int | bool]] = mp_ctx.Queue()
|
||||
|
||||
# Start both node processes
|
||||
processes = []
|
||||
for node_rank in range(2):
|
||||
p = mp_ctx.Process(
|
||||
target=run_node,
|
||||
args=(node_rank, result_queue, port),
|
||||
name=f"Node{node_rank}",
|
||||
)
|
||||
p.start()
|
||||
processes.append(p)
|
||||
|
||||
# Wait for both processes to complete
|
||||
all_completed = True
|
||||
for p in processes:
|
||||
p.join(timeout=60)
|
||||
if p.is_alive():
|
||||
p.terminate()
|
||||
p.join(timeout=20)
|
||||
if p.is_alive():
|
||||
p.kill()
|
||||
p.join()
|
||||
all_completed = False
|
||||
|
||||
# Check results from both nodes
|
||||
results: list[dict[str, int | bool]] = []
|
||||
while len(results) < 2:
|
||||
try:
|
||||
result = result_queue.get(timeout=1)
|
||||
results.append(result)
|
||||
except Exception:
|
||||
pass
|
||||
assert all_completed, "Not all processes completed successfully"
|
||||
assert len(results) == 2, f"Expected 2 results, got {len(results)}"
|
||||
assert results[0]["success"], f"Node 0 failed: {results[0]}"
|
||||
assert results[1]["success"], f"Node 1 failed: {results[1]}"
|
||||
@@ -1,367 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import random
|
||||
import typing
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
import vllm.envs as envs
|
||||
from tests.utils import ensure_current_vllm_config
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.distributed.device_communicators.cuda_communicator import (
|
||||
NCCL_DIRECT_SYMM_RS_OUTPUT_MIN_VERSION,
|
||||
CudaCommunicator,
|
||||
)
|
||||
from vllm.distributed.device_communicators.pynccl import register_nccl_symmetric_ops
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import (
|
||||
get_nccl_mem_pool,
|
||||
is_symmetric_memory_enabled,
|
||||
is_symmetric_memory_tensor,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tp_group,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
torch.manual_seed(42)
|
||||
random.seed(44)
|
||||
|
||||
test_size_elements = 4 * 1024 * 1024
|
||||
|
||||
|
||||
def test_disabled_pynccl_does_not_allocate_symmetric_buffer():
|
||||
communicator = object.__new__(CudaCommunicator)
|
||||
communicator.pynccl_comm = SimpleNamespace(disabled=True)
|
||||
|
||||
assert (
|
||||
communicator.get_symmetric_memory_buffer(
|
||||
"test", (1,), torch.float32, torch.device("cuda")
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_aiter", [False, True])
|
||||
def test_reduce_scatterv_preserves_caller_output(monkeypatch, use_aiter):
|
||||
import vllm.distributed.device_communicators.cuda_communicator as cc
|
||||
|
||||
communicator = object.__new__(CudaCommunicator)
|
||||
communicator.world_size = 2
|
||||
communicator.rank_in_group = 0
|
||||
input_ = torch.arange(12, dtype=torch.float32).view(4, 3)
|
||||
output = torch.full((2, 3), float("nan"))
|
||||
|
||||
def reduce_scatter(out, inp):
|
||||
assert out is output
|
||||
out.copy_(inp.view(2, 2, 3).sum(dim=0))
|
||||
|
||||
communicator.pynccl_comm = SimpleNamespace(
|
||||
disabled=False, nccl_version=22902, reduce_scatter=reduce_scatter
|
||||
)
|
||||
communicator._can_use_aiter_ag_rs = lambda sizes: use_aiter and sizes is None
|
||||
communicator.aiter_ar_comm = SimpleNamespace(
|
||||
should_custom_rs=lambda inp, dim: True,
|
||||
custom_reduce_scatter=lambda inp, out, dim: reduce_scatter(out, inp),
|
||||
)
|
||||
monkeypatch.setattr(cc, "should_nccl_symm_mem_ag_rs", lambda: True)
|
||||
monkeypatch.setattr(cc, "is_symmetric_memory_tensor", lambda tensor: False)
|
||||
monkeypatch.setenv("VLLM_BATCH_INVARIANT", "0")
|
||||
|
||||
# Explicit uniform sizes must not stage an ordinary input into symmetric
|
||||
# scratch, and AITER must write the supplied output after normalization.
|
||||
result = communicator.reduce_scatterv_into_output(
|
||||
input_, output, dim=0, sizes=[2, 2]
|
||||
)
|
||||
assert result.data_ptr() == output.data_ptr()
|
||||
torch.testing.assert_close(output, input_.view(2, 2, 3).sum(dim=0))
|
||||
|
||||
|
||||
def nccl_symm_mem_allreduce_worker(local_rank: int, world_size: int):
|
||||
monkeypatch = pytest.MonkeyPatch()
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
dtype = torch.bfloat16
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": "12345",
|
||||
}
|
||||
)
|
||||
|
||||
init_distributed_environment()
|
||||
with ensure_current_vllm_config():
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
|
||||
cuda_communicator = typing.cast(
|
||||
CudaCommunicator, get_tp_group().device_communicator
|
||||
)
|
||||
pynccl_comm = cuda_communicator.pynccl_comm
|
||||
if get_nccl_mem_pool() is None:
|
||||
pytest.skip(
|
||||
"NCCL allocator compilation failed (probably missing NCCL headers)."
|
||||
)
|
||||
if not is_symmetric_memory_enabled():
|
||||
pytest.skip("NCCL symmetric memory allreduce is disabled.")
|
||||
|
||||
register_nccl_symmetric_ops(pynccl_comm)
|
||||
input = torch.randint(1, 23, (test_size_elements,), dtype=dtype, device=device)
|
||||
input_clone = input.clone()
|
||||
output = torch.ops.vllm.all_reduce_symmetric_with_copy(input)
|
||||
assert output is not None
|
||||
|
||||
group = get_tp_group().device_group
|
||||
dist.all_reduce(input_clone, group=group)
|
||||
torch.testing.assert_close(output, input_clone, atol=2.5, rtol=0.1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="NCCLSymmMemAllreduce is only available for CUDA platforms.",
|
||||
)
|
||||
@pytest.mark.parametrize("world_size", [2])
|
||||
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
|
||||
def test_nccl_symm_mem_allreduce(monkeypatch: pytest.MonkeyPatch, world_size):
|
||||
if world_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
|
||||
# Enable SymmMemCommunicator
|
||||
monkeypatch.setenv("VLLM_USE_NCCL_SYMM_MEM", "1")
|
||||
monkeypatch.setenv("NCCL_NVLS_ENABLE", "1")
|
||||
monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1")
|
||||
|
||||
mp.spawn(nccl_symm_mem_allreduce_worker, args=(world_size,), nprocs=world_size)
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
def nccl_symm_mem_allgather_worker(local_rank: int, world_size: int):
|
||||
monkeypatch = pytest.MonkeyPatch()
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
dtype = torch.bfloat16
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": "12346",
|
||||
}
|
||||
)
|
||||
|
||||
init_distributed_environment()
|
||||
with ensure_current_vllm_config():
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
|
||||
cuda_communicator = typing.cast(
|
||||
CudaCommunicator, get_tp_group().device_communicator
|
||||
)
|
||||
if get_nccl_mem_pool() is None:
|
||||
pytest.skip(
|
||||
"NCCL allocator compilation failed (probably missing NCCL headers)."
|
||||
)
|
||||
if not is_symmetric_memory_enabled():
|
||||
pytest.skip("NCCL symmetric memory is disabled.")
|
||||
|
||||
per_rank_size = test_size_elements // world_size
|
||||
input_tensor = torch.randint(
|
||||
1, 23, (per_rank_size,), dtype=dtype, device=device
|
||||
)
|
||||
output = cuda_communicator.all_gatherv(input_tensor, dim=0)
|
||||
|
||||
group = get_tp_group().device_group
|
||||
expected = torch.empty(test_size_elements, dtype=dtype, device=device)
|
||||
dist.all_gather_into_tensor(expected, input_tensor, group=group)
|
||||
torch.testing.assert_close(output, expected, atol=0.0, rtol=0.0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="NCCL symmetric memory is only available for CUDA platforms.",
|
||||
)
|
||||
@pytest.mark.parametrize("world_size", [2])
|
||||
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
|
||||
def test_nccl_symm_mem_allgather(monkeypatch: pytest.MonkeyPatch, world_size):
|
||||
if world_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_NCCL_SYMM_MEM", "1")
|
||||
monkeypatch.setenv("NCCL_NVLS_ENABLE", "1")
|
||||
monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1")
|
||||
|
||||
mp.spawn(nccl_symm_mem_allgather_worker, args=(world_size,), nprocs=world_size)
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
def nccl_symm_mem_reduce_scatter_worker(local_rank: int, world_size: int):
|
||||
monkeypatch = pytest.MonkeyPatch()
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
dtype = torch.bfloat16
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": "12347",
|
||||
}
|
||||
)
|
||||
|
||||
init_distributed_environment()
|
||||
with ensure_current_vllm_config():
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
|
||||
cuda_communicator = typing.cast(
|
||||
CudaCommunicator, get_tp_group().device_communicator
|
||||
)
|
||||
if get_nccl_mem_pool() is None:
|
||||
pytest.skip(
|
||||
"NCCL allocator compilation failed (probably missing NCCL headers)."
|
||||
)
|
||||
if not is_symmetric_memory_enabled():
|
||||
pytest.skip("NCCL symmetric memory is disabled.")
|
||||
|
||||
per_rank_size = test_size_elements // world_size
|
||||
ordinary_input = torch.randint(
|
||||
1, 23, (test_size_elements,), dtype=dtype, device=device
|
||||
)
|
||||
ordinary_clone = ordinary_input.clone()
|
||||
ordinary_output = cuda_communicator.reduce_scatter(ordinary_input, dim=0)
|
||||
|
||||
group = get_tp_group().device_group
|
||||
ordinary_expected = torch.empty(per_rank_size, dtype=dtype, device=device)
|
||||
dist.reduce_scatter_tensor(ordinary_expected, ordinary_clone, group=group)
|
||||
torch.testing.assert_close(
|
||||
ordinary_output, ordinary_expected, atol=2.5, rtol=0.1
|
||||
)
|
||||
assert is_symmetric_memory_tensor(ordinary_output)
|
||||
|
||||
# Both calls reuse rs_out; validate the first result before overwriting it.
|
||||
for sizes in (None, [per_rank_size] * world_size):
|
||||
ordinary_v_output = cuda_communicator.reduce_scatterv(
|
||||
ordinary_input, dim=0, sizes=sizes
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
ordinary_v_output, ordinary_expected, atol=2.5, rtol=0.1
|
||||
)
|
||||
assert is_symmetric_memory_tensor(ordinary_v_output)
|
||||
|
||||
pynccl_comm = cuda_communicator.pynccl_comm
|
||||
assert pynccl_comm is not None
|
||||
if pynccl_comm.nccl_version < NCCL_DIRECT_SYMM_RS_OUTPUT_MIN_VERSION:
|
||||
return
|
||||
|
||||
from vllm.v1.worker import ubatching
|
||||
|
||||
m.setattr(ubatching, "dbo_current_ubatch_id", lambda: 0)
|
||||
ubatch0 = cuda_communicator._get_symm_scratch("ag_out", (128,), dtype, device)
|
||||
ubatch0.fill_(1)
|
||||
m.setattr(ubatching, "dbo_current_ubatch_id", lambda: 1)
|
||||
ubatch1 = cuda_communicator._get_symm_scratch("ag_out", (128,), dtype, device)
|
||||
ubatch1.fill_(2)
|
||||
assert ubatch0.data_ptr() != ubatch1.data_ptr()
|
||||
assert (ubatch0 == 1).all()
|
||||
|
||||
m.setattr(ubatching, "dbo_current_ubatch_id", lambda: 0)
|
||||
smaller = cuda_communicator._get_symm_scratch("ag_out", (64,), dtype, device)
|
||||
assert smaller.shape == (64,)
|
||||
assert smaller.data_ptr() == ubatch0.data_ptr()
|
||||
|
||||
grown = cuda_communicator._get_symm_scratch("ag_out", (129,), dtype, device)
|
||||
assert grown.shape == (129,)
|
||||
assert grown.data_ptr() != ubatch0.data_ptr()
|
||||
reused = cuda_communicator._get_symm_scratch("ag_out", (200,), dtype, device)
|
||||
assert reused.data_ptr() == grown.data_ptr()
|
||||
|
||||
input_tensor = cuda_communicator._get_symm_scratch(
|
||||
"test_graph_rs_input", (test_size_elements,), dtype, device
|
||||
)
|
||||
input_tensor.random_(1, 23)
|
||||
input_clone = input_tensor.clone()
|
||||
output = torch.empty(per_rank_size, dtype=dtype, device=device)
|
||||
assert is_symmetric_memory_tensor(input_tensor)
|
||||
assert not is_symmetric_memory_tensor(output)
|
||||
|
||||
result = cuda_communicator.reduce_scatterv_into_output(
|
||||
input_tensor,
|
||||
output,
|
||||
dim=0,
|
||||
sizes=[per_rank_size] * world_size,
|
||||
)
|
||||
assert result.data_ptr() == output.data_ptr()
|
||||
|
||||
expected = torch.empty_like(output)
|
||||
dist.reduce_scatter_tensor(expected, input_clone, group=group)
|
||||
torch.testing.assert_close(result, expected, atol=2.5, rtol=0.1)
|
||||
|
||||
dist.barrier(group=get_tp_group().cpu_group)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
graph_result = cuda_communicator.reduce_scatterv_into_output(
|
||||
input_tensor,
|
||||
output,
|
||||
dim=0,
|
||||
sizes=[per_rank_size] * world_size,
|
||||
)
|
||||
assert graph_result.data_ptr() == output.data_ptr()
|
||||
|
||||
input_tensor.random_(24, 47)
|
||||
input_clone.copy_(input_tensor)
|
||||
dist.reduce_scatter_tensor(expected, input_clone, group=group)
|
||||
captured_input_ptr = input_tensor.data_ptr()
|
||||
del input_tensor
|
||||
grown_input = cuda_communicator._get_symm_scratch(
|
||||
"test_graph_rs_input", (test_size_elements + 1,), dtype, device
|
||||
)
|
||||
assert grown_input.data_ptr() != captured_input_ptr
|
||||
retired = cuda_communicator.__dict__["_retired_symm_scratch_bufs"]
|
||||
assert any(
|
||||
buf.data_ptr() == captured_input_ptr
|
||||
for buffers in retired.values()
|
||||
for buf in buffers
|
||||
)
|
||||
output.zero_()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(output, expected, atol=2.5, rtol=0.1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="NCCL symmetric memory is only available for CUDA platforms.",
|
||||
)
|
||||
@pytest.mark.parametrize("world_size", [2])
|
||||
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
|
||||
def test_nccl_symm_mem_reduce_scatter(monkeypatch: pytest.MonkeyPatch, world_size):
|
||||
if world_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_NCCL_SYMM_MEM", "1")
|
||||
monkeypatch.setenv("NCCL_NVLS_ENABLE", "1")
|
||||
monkeypatch.setenv("NCCL_CUMEM_ENABLE", "1")
|
||||
|
||||
mp.spawn(nccl_symm_mem_reduce_scatter_worker, args=(world_size,), nprocs=world_size)
|
||||
cleanup_dist_env_and_memory()
|
||||
@@ -1,41 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import compare_two_settings, create_new_process_for_each_test
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"PP_SIZE, MODEL_NAME",
|
||||
[
|
||||
(2, "JackFram/llama-160m"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"ATTN_BACKEND",
|
||||
[None] if current_platform.is_rocm() else ["FLASH_ATTN"],
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
def test_pp_cudagraph(
|
||||
PP_SIZE: int,
|
||||
MODEL_NAME: str,
|
||||
ATTN_BACKEND: str | None,
|
||||
):
|
||||
cudagraph_args = [
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"float16",
|
||||
"--pipeline-parallel-size",
|
||||
str(PP_SIZE),
|
||||
"--distributed-executor-backend",
|
||||
"mp",
|
||||
]
|
||||
# On ROCm, defer to the platform attention selector instead of forcing a backend.
|
||||
if ATTN_BACKEND is not None:
|
||||
cudagraph_args.append(f"--attention-backend={ATTN_BACKEND}")
|
||||
|
||||
eager_args = cudagraph_args + ["--enforce-eager"]
|
||||
|
||||
compare_two_settings(MODEL_NAME, eager_args, cudagraph_args)
|
||||
@@ -1,664 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import multiprocessing
|
||||
import random
|
||||
import sys
|
||||
import types
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import vllm._aiter_ops as aiter_ops
|
||||
import vllm.distributed as vllm_distributed
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import (
|
||||
KB,
|
||||
MB,
|
||||
QuickAllReduce,
|
||||
QuickReduceRegime,
|
||||
)
|
||||
from vllm.distributed.parallel_state import get_tp_group, graph_capture
|
||||
from vllm.envs import disable_envs_cache
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_test_distributed_environment,
|
||||
multi_process_parallel,
|
||||
set_random_seed,
|
||||
)
|
||||
|
||||
|
||||
def on_gfx942() -> bool:
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx942 as rocm_on_gfx942
|
||||
|
||||
return rocm_on_gfx942()
|
||||
return False
|
||||
|
||||
|
||||
def _quickreduce_tolerance(quant_level: QuickReduceRegime) -> dict:
|
||||
# With INT3 inputs of absmax 23, per-rank error peaks at 2.75,
|
||||
# doubled by the TP2 sum = 5.5.
|
||||
if quant_level == QuickReduceRegime.INT3:
|
||||
return {"atol": 5.5, "rtol": 0.1}
|
||||
return {"atol": 2.5, "rtol": 0.1}
|
||||
|
||||
|
||||
set_random_seed(42)
|
||||
_test_size_rng = random.Random(44)
|
||||
# Size over 8MB is sufficient for custom quick allreduce.
|
||||
test_sizes = [
|
||||
_test_size_rng.randint(8 * 1024 * 1024, 10 * 1024 * 1024) for _ in range(8)
|
||||
]
|
||||
for i, v in enumerate(test_sizes):
|
||||
test_sizes[i] -= v % 8
|
||||
|
||||
|
||||
def _assert_quickreduce(fa, inp):
|
||||
assert fa is not None
|
||||
assert not fa.disabled
|
||||
assert fa.should_quick_allreduce(inp)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def envs_cache_disabled():
|
||||
disable_envs_cache()
|
||||
yield
|
||||
disable_envs_cache()
|
||||
|
||||
|
||||
def _make_quick_allreduce_for_test(
|
||||
min_size_mb: int | None = None,
|
||||
quantization_min_size: int | None = None,
|
||||
) -> QuickAllReduce:
|
||||
quick_reduce = QuickAllReduce.__new__(QuickAllReduce)
|
||||
quick_reduce.disabled = False
|
||||
quick_reduce.qr_max_size = 16 * MB
|
||||
quick_reduce.qr_min_size = min_size_mb * MB if min_size_mb is not None else None
|
||||
quick_reduce.qr_quant_level = QuickReduceRegime.INT4
|
||||
quick_reduce.qr_quantization_min_size = quantization_min_size
|
||||
quick_reduce.use_fp16_kernels = False
|
||||
quick_reduce.world_size = 2
|
||||
return quick_reduce
|
||||
|
||||
|
||||
def test_should_quick_allreduce_uses_builtin_min_size_when_unset():
|
||||
quick_reduce = _make_quick_allreduce_for_test(min_size_mb=None)
|
||||
|
||||
below_builtin_min = torch.empty(MB // 4, dtype=torch.float16)
|
||||
at_builtin_min = torch.empty(MB // 2, dtype=torch.float16)
|
||||
|
||||
assert not quick_reduce.should_quick_allreduce(below_builtin_min)
|
||||
assert quick_reduce.should_quick_allreduce(at_builtin_min)
|
||||
|
||||
|
||||
def test_should_quick_allreduce_uses_min_size_override():
|
||||
quick_reduce = _make_quick_allreduce_for_test(min_size_mb=0)
|
||||
|
||||
below_builtin_min = torch.empty(8, dtype=torch.float16)
|
||||
|
||||
assert quick_reduce.should_quick_allreduce(below_builtin_min)
|
||||
|
||||
|
||||
def test_quick_allreduce_min_size_env_unset(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
envs_cache_disabled,
|
||||
):
|
||||
monkeypatch.delenv("VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB", raising=False)
|
||||
|
||||
assert QuickAllReduce._get_qr_min_size(qr_max_size=16 * MB) is None
|
||||
|
||||
|
||||
def test_quick_allreduce_min_size_env_converts_mb_to_bytes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
envs_cache_disabled,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB", "4")
|
||||
|
||||
assert QuickAllReduce._get_qr_min_size(qr_max_size=16 * MB) == 4 * MB
|
||||
|
||||
|
||||
def test_quick_allreduce_min_size_env_rejects_negative(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
envs_cache_disabled,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB", "-1")
|
||||
|
||||
with pytest.raises(ValueError, match="must be non-negative"):
|
||||
QuickAllReduce._get_qr_min_size(qr_max_size=16 * MB)
|
||||
|
||||
|
||||
def test_quick_allreduce_min_size_env_allows_equal_to_max(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
envs_cache_disabled,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB", "16")
|
||||
|
||||
assert QuickAllReduce._get_qr_min_size(qr_max_size=16 * MB) == 16 * MB
|
||||
|
||||
|
||||
def test_quick_allreduce_min_size_env_rejects_larger_than_max(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
envs_cache_disabled,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_MIN_SIZE_BYTES_MB", "17")
|
||||
|
||||
with pytest.raises(ValueError, match="effective QuickReduce max size"):
|
||||
QuickAllReduce._get_qr_min_size(qr_max_size=16 * MB)
|
||||
|
||||
|
||||
def test_quick_allreduce_quantization_min_size_env_unset(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
envs_cache_disabled,
|
||||
):
|
||||
monkeypatch.delenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB", raising=False)
|
||||
|
||||
assert QuickAllReduce._get_qr_quantization_min_size() is None
|
||||
|
||||
|
||||
def test_quick_allreduce_quantization_min_size_env_converts_kb_to_bytes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
envs_cache_disabled,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB", "2048")
|
||||
|
||||
assert QuickAllReduce._get_qr_quantization_min_size() == 2048 * KB
|
||||
|
||||
|
||||
def test_quick_allreduce_quantization_min_size_env_rejects_negative(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
envs_cache_disabled,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB", "-1")
|
||||
|
||||
with pytest.raises(ValueError, match="must be non-negative"):
|
||||
QuickAllReduce._get_qr_quantization_min_size()
|
||||
|
||||
|
||||
def test_quick_allreduce_quantization_min_size_unset_uses_configured_codec():
|
||||
quick_reduce = _make_quick_allreduce_for_test(quantization_min_size=None)
|
||||
inp = torch.empty(8, dtype=torch.float16)
|
||||
|
||||
assert quick_reduce._get_qr_quant_level(inp) == QuickReduceRegime.INT4.value
|
||||
|
||||
|
||||
def test_quick_allreduce_quantization_min_size_uses_fp_below_threshold():
|
||||
quick_reduce = _make_quick_allreduce_for_test(quantization_min_size=2048)
|
||||
inp = torch.empty(1024 // 2, dtype=torch.float16)
|
||||
|
||||
assert quick_reduce._get_qr_quant_level(inp) == QuickReduceRegime.FP.value
|
||||
|
||||
|
||||
def test_quick_allreduce_quantization_min_size_uses_configured_codec_at_threshold():
|
||||
quick_reduce = _make_quick_allreduce_for_test(quantization_min_size=2048)
|
||||
inp = torch.empty(2048 // 2, dtype=torch.float16)
|
||||
|
||||
assert quick_reduce._get_qr_quant_level(inp) == QuickReduceRegime.INT4.value
|
||||
|
||||
|
||||
def test_quick_allreduce_quantization_min_size_does_not_change_eligibility():
|
||||
quick_reduce = _make_quick_allreduce_for_test(quantization_min_size=2 * MB)
|
||||
|
||||
below_builtin_min = torch.empty(MB // 4, dtype=torch.float16)
|
||||
at_builtin_min = torch.empty(MB // 2, dtype=torch.float16)
|
||||
|
||||
assert not quick_reduce.should_quick_allreduce(below_builtin_min)
|
||||
assert quick_reduce.should_quick_allreduce(at_builtin_min)
|
||||
|
||||
|
||||
def test_quick_allreduce_passes_dynamic_quant_level(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
quick_reduce = _make_quick_allreduce_for_test(quantization_min_size=2 * KB)
|
||||
quick_reduce._ptr = object()
|
||||
inp = torch.empty(KB // 2, dtype=torch.float16)
|
||||
called_quant_level = None
|
||||
|
||||
def fake_qr_all_reduce(
|
||||
fa,
|
||||
inp,
|
||||
out,
|
||||
quant_level,
|
||||
cast_bf2half,
|
||||
):
|
||||
nonlocal called_quant_level
|
||||
called_quant_level = quant_level
|
||||
|
||||
monkeypatch.setattr(ops, "qr_all_reduce", fake_qr_all_reduce)
|
||||
|
||||
quick_reduce.quick_all_reduce(inp)
|
||||
|
||||
assert called_quant_level == QuickReduceRegime.FP.value
|
||||
|
||||
|
||||
def test_rocm_aiter_fused_rmsnorm_uses_aiter_qr_rmsnorm_for_prefill(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
class FakeAiterAllReduce:
|
||||
aiter_ca = SimpleNamespace(world_size=2, fully_connected=True)
|
||||
|
||||
def use_1stage_fused_ar_rms(self, inp):
|
||||
return False
|
||||
|
||||
def custom_fused_ar_rms(self, *args, **kwargs):
|
||||
raise AssertionError("expected fused QR+RMSNorm dispatch")
|
||||
|
||||
class FakeQuickReduce:
|
||||
disabled = False
|
||||
|
||||
def __init__(self):
|
||||
self.checked_input = None
|
||||
|
||||
def should_quick_allreduce(self, inp):
|
||||
self.checked_input = inp
|
||||
return True
|
||||
|
||||
def _get_qr_quant_level(self, inp):
|
||||
return QuickReduceRegime.INT4.value
|
||||
|
||||
qr_comm = FakeQuickReduce()
|
||||
device_comm = SimpleNamespace(cpu_group=object(), qr_comm=qr_comm)
|
||||
monkeypatch.setattr(
|
||||
vllm_distributed,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(device_communicator=device_comm),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
aiter_ops.rocm_aiter_ops,
|
||||
"get_aiter_allreduce",
|
||||
lambda: FakeAiterAllReduce(),
|
||||
)
|
||||
monkeypatch.setattr(aiter_ops.dist, "get_world_size", lambda group: 2)
|
||||
monkeypatch.setattr(aiter_ops.dist, "get_rank", lambda group: 0)
|
||||
monkeypatch.setattr(
|
||||
aiter_ops.dist,
|
||||
"all_gather_object",
|
||||
lambda handles, handle, group: handles.__setitem__(0, handle),
|
||||
)
|
||||
|
||||
calls = []
|
||||
fake_aiter = types.SimpleNamespace(
|
||||
init_custom_qr=lambda rank, world_size, qr_max_size: 123,
|
||||
qr_get_handle=lambda ptr: b"handle",
|
||||
qr_open_handles=lambda ptr, handles: None,
|
||||
)
|
||||
|
||||
def fake_qr_all_reduce_rmsnorm(
|
||||
ptr,
|
||||
inp,
|
||||
residual,
|
||||
residual_out,
|
||||
out,
|
||||
weight,
|
||||
eps,
|
||||
hidden_dim,
|
||||
quant_level,
|
||||
cast_bf2half,
|
||||
):
|
||||
calls.append(
|
||||
{
|
||||
"ptr": ptr,
|
||||
"inp": inp,
|
||||
"residual": residual,
|
||||
"residual_out": residual_out,
|
||||
"out": out,
|
||||
"weight": weight,
|
||||
"eps": eps,
|
||||
"hidden_dim": hidden_dim,
|
||||
"quant_level": quant_level,
|
||||
"cast_bf2half": cast_bf2half,
|
||||
}
|
||||
)
|
||||
out.copy_(inp)
|
||||
residual_out.copy_(residual)
|
||||
|
||||
fake_aiter.qr_all_reduce_rmsnorm = fake_qr_all_reduce_rmsnorm
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(envs, "VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", None)
|
||||
monkeypatch.setattr(envs, "VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", True)
|
||||
|
||||
# token_num > 80 forces vLLM's AITER fused AR+RMS wrapper onto its
|
||||
# non-1stage prefill path. hidden_dim=4096 bf16 gives an 8 KiB row, so four
|
||||
# independent RMSNorm rows fit in one 32 KiB QR tile.
|
||||
inp = torch.randn(81, 4096, dtype=torch.bfloat16)
|
||||
residual = torch.randn_like(inp)
|
||||
weight = torch.randn(4096, dtype=torch.bfloat16)
|
||||
|
||||
out, residual_out = aiter_ops._rocm_aiter_fused_allreduce_rmsnorm_impl(
|
||||
inp, residual, weight, 1e-6
|
||||
)
|
||||
|
||||
assert qr_comm.checked_input is inp
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["ptr"] == 123
|
||||
assert calls[0]["inp"] is inp
|
||||
assert calls[0]["residual"] is residual
|
||||
assert calls[0]["out"] is out
|
||||
assert calls[0]["residual_out"] is residual_out
|
||||
assert calls[0]["weight"] is weight
|
||||
assert calls[0]["hidden_dim"] == 4096
|
||||
assert calls[0]["quant_level"] == QuickReduceRegime.INT4.value
|
||||
assert calls[0]["cast_bf2half"] is True
|
||||
torch.testing.assert_close(out, inp)
|
||||
torch.testing.assert_close(residual_out, residual)
|
||||
|
||||
|
||||
def test_rocm_aiter_fused_rmsnorm_keeps_1stage_decode_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
calls = []
|
||||
|
||||
class FakeAiterCA:
|
||||
world_size = 2
|
||||
fully_connected = True
|
||||
|
||||
def custom_fused_ar_rms(
|
||||
self, inp, residual, weight, epsilon, *, use_1stage, gemma_norm
|
||||
):
|
||||
calls.append(
|
||||
{
|
||||
"inp": inp,
|
||||
"residual": residual,
|
||||
"weight": weight,
|
||||
"eps": epsilon,
|
||||
"use_1stage": use_1stage,
|
||||
"gemma_norm": gemma_norm,
|
||||
}
|
||||
)
|
||||
return inp + 1, residual + 1
|
||||
|
||||
class FakeAiterAllReduce:
|
||||
aiter_ca = FakeAiterCA()
|
||||
|
||||
def use_1stage_fused_ar_rms(self, inp):
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(
|
||||
aiter_ops.rocm_aiter_ops,
|
||||
"get_aiter_allreduce",
|
||||
lambda: FakeAiterAllReduce(),
|
||||
)
|
||||
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
|
||||
|
||||
inp = torch.randn(4, 4096, dtype=torch.bfloat16)
|
||||
residual = torch.randn_like(inp)
|
||||
weight = torch.randn(4096, dtype=torch.bfloat16)
|
||||
|
||||
out, residual_out = aiter_ops._rocm_aiter_fused_allreduce_rmsnorm_impl(
|
||||
inp, residual, weight, 1e-6
|
||||
)
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["inp"] is inp
|
||||
assert calls[0]["residual"] is residual
|
||||
assert calls[0]["weight"] is weight
|
||||
assert calls[0]["use_1stage"] is True
|
||||
assert calls[0]["gemma_norm"] is False
|
||||
torch.testing.assert_close(out, inp + 1)
|
||||
torch.testing.assert_close(residual_out, residual + 1)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def graph_quickreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("ROCR_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
group = get_tp_group().device_group
|
||||
fa = get_tp_group().device_communicator.qr_comm
|
||||
|
||||
# A small all_reduce for warmup.
|
||||
# this is needed because device communicators might be created lazily
|
||||
# (e.g. NCCL). This will ensure that the communicator is initialized
|
||||
# before any communication happens, so that this group can be used for
|
||||
# graph capture immediately.
|
||||
data = torch.zeros(1)
|
||||
data = data.to(device=device)
|
||||
torch.distributed.all_reduce(data, group=group)
|
||||
torch.accelerator.synchronize()
|
||||
del data
|
||||
|
||||
# we use the first group to communicate once
|
||||
# and the second group to communicate twice
|
||||
# and so on
|
||||
# this is used to demonstrate that each group can
|
||||
# communicate independently
|
||||
num_communication = rank // tp_size + 1
|
||||
|
||||
for sz in test_sizes:
|
||||
for dtype in [torch.float16, torch.bfloat16]:
|
||||
with graph_capture(device=device) as graph_capture_context:
|
||||
device_idx = torch.accelerator.current_device_index()
|
||||
inp1 = torch.randint(1, 23, (sz,), dtype=dtype, device=device_idx)
|
||||
inp2 = torch.randint(-23, 1, (sz,), dtype=dtype, device=device_idx)
|
||||
_assert_quickreduce(fa, inp1)
|
||||
_assert_quickreduce(fa, inp2)
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
|
||||
for _ in range(num_communication):
|
||||
out1 = tensor_model_parallel_all_reduce(inp1)
|
||||
dist.all_reduce(inp1, group=group)
|
||||
out2 = tensor_model_parallel_all_reduce(inp2)
|
||||
dist.all_reduce(inp2, group=group)
|
||||
graph.replay()
|
||||
tol = _quickreduce_tolerance(fa.qr_quant_level)
|
||||
torch.testing.assert_close(out1, inp1, **tol)
|
||||
torch.testing.assert_close(out2, inp2, **tol)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def eager_quickreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("ROCR_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
|
||||
# Size over 8MB is sufficient for custom quick allreduce.
|
||||
sz = 16 * 1024 * 1024
|
||||
fa = get_tp_group().device_communicator.qr_comm
|
||||
inp = torch.tensor(
|
||||
[1.0 * ((i) % 23) for i in range(sz)], dtype=torch.float16, device=device
|
||||
)
|
||||
_assert_quickreduce(fa, inp)
|
||||
out = fa.quick_all_reduce(inp)
|
||||
tol = _quickreduce_tolerance(fa.qr_quant_level)
|
||||
torch.testing.assert_close(out, inp * tp_size, **tol)
|
||||
|
||||
inp = torch.tensor(
|
||||
[1.0 * ((i) % 23) for i in range(sz)], dtype=torch.bfloat16, device=device
|
||||
)
|
||||
_assert_quickreduce(fa, inp)
|
||||
out = fa.quick_all_reduce(inp)
|
||||
torch.testing.assert_close(out, inp * tp_size, **tol)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def bf16_cast_quickreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("ROCR_VISIBLE_DEVICES", raising=False)
|
||||
m.setenv("VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", "1")
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
|
||||
sz = 16 * 1024 * 1024
|
||||
fa = get_tp_group().device_communicator.qr_comm
|
||||
inp = torch.tensor(
|
||||
[1.0 * (i % 23) for i in range(sz)], dtype=torch.bfloat16, device=device
|
||||
)
|
||||
_assert_quickreduce(fa, inp)
|
||||
assert fa.use_fp16_kernels
|
||||
out = fa.quick_all_reduce(inp)
|
||||
torch.testing.assert_close(out, inp * tp_size, atol=2.5, rtol=0.1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="only test quick allreduce for rocm"
|
||||
)
|
||||
@pytest.mark.parametrize("quant_mode", ["FP", "INT8", "INT6", "INT4", "INT3"])
|
||||
@pytest.mark.parametrize("tp_size", [2])
|
||||
@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
|
||||
@pytest.mark.parametrize("test_target", [graph_quickreduce, eager_quickreduce])
|
||||
def test_custom_quick_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pipeline_parallel_size,
|
||||
test_target,
|
||||
quant_mode,
|
||||
):
|
||||
world_size = tp_size * pipeline_parallel_size
|
||||
if world_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
if test_target is graph_quickreduce and on_gfx942():
|
||||
pytest.xfail(
|
||||
"CUDA graph capture with quick reduce hits "
|
||||
"hipErrorStreamCaptureInvalidated on gfx942"
|
||||
)
|
||||
|
||||
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", quant_mode)
|
||||
|
||||
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="only test quick allreduce for rocm"
|
||||
)
|
||||
def test_custom_quick_allreduce_bf16_cast(monkeypatch: pytest.MonkeyPatch):
|
||||
if torch.accelerator.device_count() < 2:
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", "FP")
|
||||
multi_process_parallel(monkeypatch, 2, 1, bf16_cast_quickreduce)
|
||||
|
||||
|
||||
def qr_variable_input(rank, world_size):
|
||||
"""When the tensor parallelism is set to 4 or 8, frequent changes
|
||||
in the input shape can cause QuickReduce to hang (this issue
|
||||
has been observed with the gpt_oss model).
|
||||
"""
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
qr_max_size = None # MB
|
||||
_ptr = ops.init_custom_qr(rank, world_size, qr_max_size)
|
||||
ranks = []
|
||||
for i in range(world_size):
|
||||
ranks.append(i)
|
||||
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
|
||||
dist.init_process_group(
|
||||
backend="cpu:gloo,cuda:nccl",
|
||||
init_method="tcp://127.0.0.1:29500",
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
device_id=device,
|
||||
)
|
||||
else:
|
||||
dist.init_process_group(
|
||||
backend="nccl",
|
||||
init_method="tcp://127.0.0.1:29500",
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
)
|
||||
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
|
||||
cpu_group = torch.distributed.split_group(
|
||||
split_ranks=[ranks], backend="cpu:gloo,cuda:nccl"
|
||||
)
|
||||
else:
|
||||
cpu_group = torch.distributed.new_group(ranks, backend="nccl")
|
||||
|
||||
handle = ops.qr_get_handle(_ptr)
|
||||
world_size = dist.get_world_size(group=cpu_group)
|
||||
handles = [None] * world_size
|
||||
dist.all_gather_object(handles, handle, group=cpu_group)
|
||||
ops.qr_open_handles(_ptr, handles)
|
||||
|
||||
num = 1
|
||||
s1 = 1024
|
||||
while num < 50000: # 50000 is sufficient to identify issues.
|
||||
dtype = torch.float16
|
||||
device_idx = torch.accelerator.current_device_index()
|
||||
if num % 2 == 0:
|
||||
s2 = 1024
|
||||
inp1 = torch.zeros((s1, s2), dtype=dtype, device=device_idx)
|
||||
else:
|
||||
s2 = 2048
|
||||
inp1 = torch.ones((s1, s2), dtype=dtype, device=device_idx)
|
||||
result = torch.empty_like(inp1)
|
||||
# FP = 0 INT8 = 1 INT6 = 2 INT4 = 3 INT3 = 4
|
||||
ops.qr_all_reduce(_ptr, inp1, result, 3, cast_bf2half=True)
|
||||
try:
|
||||
if inp1[0, 0] == 0:
|
||||
assert torch.all(result == 0)
|
||||
else:
|
||||
assert torch.all(result == world_size)
|
||||
except AssertionError:
|
||||
print("Assertion failed! Allreduce results are incorrect.")
|
||||
raise
|
||||
num += 1
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="only test quick allreduce for rocm"
|
||||
)
|
||||
@pytest.mark.parametrize("tp_size", [4, 8])
|
||||
@pytest.mark.parametrize("pipeline_parallel_size", [1])
|
||||
def test_custom_quick_allreduce_variable_input(tp_size, pipeline_parallel_size):
|
||||
world_size = tp_size * pipeline_parallel_size
|
||||
if world_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
|
||||
multiprocessing.set_start_method("spawn", force=True)
|
||||
# 60s is enough
|
||||
timeout = 60
|
||||
processes = []
|
||||
for rank in range(tp_size):
|
||||
p = multiprocessing.Process(target=qr_variable_input, args=(rank, tp_size))
|
||||
p.start()
|
||||
processes.append((rank, p))
|
||||
for rank, p in processes:
|
||||
p.join(timeout=timeout)
|
||||
if p.is_alive():
|
||||
for r, proc in processes:
|
||||
if proc.is_alive():
|
||||
proc.terminate()
|
||||
proc.join()
|
||||
raise RuntimeError(f"QuickReduce hang detected after {timeout} seconds!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_custom_quick_allreduce_variable_input(tp_size=4, pipeline_parallel_size=1)
|
||||
@@ -1,407 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Integration tests for RayExecutorV2 at the executor level.
|
||||
Validates executor initialization, placement group support, RPC calls,
|
||||
and distributed execution with various TP/PP configurations.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import gc
|
||||
import socket
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
from torch.distributed import TCPStore
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.v1.executor import ray_executor_v2
|
||||
from vllm.v1.executor.ray_executor_v2 import RayExecutorV2
|
||||
|
||||
pytestmark = pytest.mark.usefixtures("enable_ray_v2_backend")
|
||||
|
||||
MODEL = "facebook/opt-125m"
|
||||
|
||||
|
||||
def create_vllm_config(
|
||||
tensor_parallel_size: int = 1,
|
||||
pipeline_parallel_size: int = 1,
|
||||
max_model_len: int = 256,
|
||||
gpu_memory_utilization: float = 0.3,
|
||||
placement_group=None,
|
||||
) -> VllmConfig:
|
||||
engine_args = EngineArgs(
|
||||
model=MODEL,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
pipeline_parallel_size=pipeline_parallel_size,
|
||||
max_model_len=max_model_len,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
distributed_executor_backend="ray",
|
||||
enforce_eager=True,
|
||||
)
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
|
||||
if placement_group is not None:
|
||||
vllm_config.parallel_config.placement_group = placement_group
|
||||
|
||||
return vllm_config
|
||||
|
||||
|
||||
def ensure_ray_initialized():
|
||||
if not ray.is_initialized():
|
||||
ray.init(ignore_reinit_error=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def create_placement_group(request):
|
||||
ensure_ray_initialized()
|
||||
num_gpus = request.param
|
||||
bundles = [{"GPU": 1, "CPU": 1} for _ in range(num_gpus)]
|
||||
pg = ray.util.placement_group(bundles, strategy="PACK")
|
||||
ray.get(pg.ready())
|
||||
yield pg
|
||||
ray.util.remove_placement_group(pg)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def executor(request):
|
||||
"""Create a RayExecutorV2 and shut it down after the test."""
|
||||
executor = RayExecutorV2(vllm_config=request.param)
|
||||
yield executor
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
def assert_executor(executor, tp_size, pp_size):
|
||||
"""Common assertions for executor initialization tests."""
|
||||
world_size = tp_size * pp_size
|
||||
expected_output_rank = (pp_size - 1) * tp_size
|
||||
|
||||
assert executor.world_size == world_size
|
||||
assert len(executor.ray_worker_handles) == world_size
|
||||
assert len(executor.response_mqs) == world_size
|
||||
assert executor._get_output_rank() == expected_output_rank
|
||||
|
||||
if pp_size > 1:
|
||||
expected_concurrent_batches = pp_size + int(
|
||||
executor.vllm_config.scheduler_config.async_scheduling
|
||||
and executor.vllm_config.use_v2_model_runner
|
||||
)
|
||||
assert (
|
||||
executor.vllm_config.max_concurrent_batches == expected_concurrent_batches
|
||||
)
|
||||
|
||||
executor.check_health()
|
||||
assert not executor.is_failed
|
||||
|
||||
ranks = sorted(h.rank for h in executor.ray_worker_handles)
|
||||
assert ranks == list(range(world_size))
|
||||
|
||||
for handle in executor.ray_worker_handles:
|
||||
assert handle.node_id is not None
|
||||
|
||||
|
||||
def _detached_rank0_worker() -> ray_executor_v2.RayWorkerProc:
|
||||
"""A RayWorkerProc that can create its dist-init store without Ray."""
|
||||
worker = ray_executor_v2.RayWorkerProc.__new__(ray_executor_v2.RayWorkerProc)
|
||||
worker._parallel_config = SimpleNamespace(world_size=1)
|
||||
return worker
|
||||
|
||||
|
||||
def _assert_port_held(port: int):
|
||||
with socket.socket() as contender, pytest.raises(OSError) as exc_info:
|
||||
contender.bind(("127.0.0.1", port))
|
||||
assert exc_info.value.errno == errno.EADDRINUSE
|
||||
|
||||
|
||||
def test_dist_init_store_owns_port(monkeypatch):
|
||||
"""The rank-0 actor keeps the selected TCPStore port bound."""
|
||||
monkeypatch.setattr(ray.util, "get_node_ip_address", lambda: "127.0.0.1")
|
||||
worker = _detached_rank0_worker()
|
||||
|
||||
init_method = worker.create_dist_init_method()
|
||||
port = urlsplit(init_method).port
|
||||
assert port is not None and port != 0
|
||||
_assert_port_held(port)
|
||||
|
||||
reused_store = TCPStore(
|
||||
"127.0.0.1",
|
||||
port,
|
||||
world_size=1,
|
||||
is_master=True,
|
||||
wait_for_workers=False,
|
||||
multi_tenant=True,
|
||||
)
|
||||
worker._dist_init_store.set("key", "value")
|
||||
assert reused_store.get("key") == b"value"
|
||||
|
||||
|
||||
def test_colocated_dist_init_stores_hold_distinct_ports(monkeypatch):
|
||||
"""Replicas sharing a host must never publish the same TCPStore port.
|
||||
|
||||
Each rank-0 store binds a kernel-assigned ephemeral port and keeps it
|
||||
for the actor's lifetime, so concurrently live replicas cannot collide.
|
||||
"""
|
||||
monkeypatch.setattr(ray.util, "get_node_ip_address", lambda: "127.0.0.1")
|
||||
workers = [_detached_rank0_worker() for _ in range(3)]
|
||||
|
||||
ports = [urlsplit(w.create_dist_init_method()).port for w in workers]
|
||||
assert len(set(ports)) == len(workers)
|
||||
for worker, port in zip(workers, ports):
|
||||
assert port == worker._dist_init_store.port
|
||||
assert port >= 1024
|
||||
_assert_port_held(port)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size, pp_size", [(1, 1), (2, 1), (4, 1), (2, 2)])
|
||||
def test_ray_v2_executor(tp_size, pp_size):
|
||||
"""Validate RayExecutorV2 with various TP/PP configs."""
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=tp_size,
|
||||
pipeline_parallel_size=pp_size,
|
||||
)
|
||||
executor = RayExecutorV2(vllm_config=vllm_config)
|
||||
try:
|
||||
assert_executor(executor, tp_size, pp_size)
|
||||
finally:
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size, pp_size, create_placement_group",
|
||||
[(2, 1, 2), (4, 1, 4), (2, 2, 4)],
|
||||
indirect=["create_placement_group"],
|
||||
)
|
||||
def test_ray_v2_executor_pg(tp_size, pp_size, create_placement_group):
|
||||
"""Validate RayExecutorV2 with various TP/PP configs using external PG."""
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=tp_size,
|
||||
pipeline_parallel_size=pp_size,
|
||||
placement_group=create_placement_group,
|
||||
)
|
||||
executor = RayExecutorV2(vllm_config=vllm_config)
|
||||
try:
|
||||
assert_executor(executor, tp_size, pp_size)
|
||||
finally:
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"executor",
|
||||
[create_vllm_config(tensor_parallel_size=2)],
|
||||
indirect=True,
|
||||
)
|
||||
def test_ray_v2_executor_failure_callback(executor):
|
||||
"""Validate failure callback registration."""
|
||||
callback_invoked = False
|
||||
|
||||
def test_callback():
|
||||
nonlocal callback_invoked
|
||||
callback_invoked = True
|
||||
|
||||
executor.register_failure_callback(test_callback)
|
||||
assert not callback_invoked
|
||||
|
||||
executor.is_failed = True
|
||||
executor.register_failure_callback(test_callback)
|
||||
assert callback_invoked
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"executor",
|
||||
[create_vllm_config(tensor_parallel_size=2)],
|
||||
indirect=True,
|
||||
)
|
||||
def test_ray_v2_executor_collective_rpc(executor):
|
||||
"""Validate collective RPC calls through MessageQueue."""
|
||||
executor.check_health()
|
||||
assert not executor.is_failed
|
||||
assert executor.rpc_broadcast_mq is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"executor",
|
||||
[create_vllm_config(tensor_parallel_size=2)],
|
||||
indirect=True,
|
||||
)
|
||||
def test_ray_v2_executor_driver_node_rank_0(executor):
|
||||
"""Validate that driver node workers get the lowest ranks."""
|
||||
driver_node = ray.get_runtime_context().get_node_id()
|
||||
|
||||
for handle in executor.ray_worker_handles:
|
||||
assert handle.node_id == driver_node
|
||||
|
||||
rank0_handle = next(h for h in executor.ray_worker_handles if h.rank == 0)
|
||||
assert rank0_handle.node_id == driver_node
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"executor",
|
||||
[create_vllm_config(tensor_parallel_size=2)],
|
||||
indirect=True,
|
||||
)
|
||||
def test_ray_v2_executor_worker_death(executor):
|
||||
"""Validate executor detects worker death via ray.wait()."""
|
||||
callback_event = threading.Event()
|
||||
|
||||
def on_failure():
|
||||
callback_event.set()
|
||||
|
||||
executor.register_failure_callback(on_failure)
|
||||
assert not executor.is_failed
|
||||
|
||||
# Kill one worker actor externally
|
||||
victim = executor.ray_worker_handles[1].actor
|
||||
ray.kill(victim, no_restart=True)
|
||||
|
||||
# Monitor thread should detect the death and invoke callback
|
||||
assert callback_event.wait(timeout=30)
|
||||
assert executor.is_failed
|
||||
assert executor.shutting_down
|
||||
|
||||
|
||||
def test_ray_v2_executor_shutdown():
|
||||
"""Validate graceful shutdown: ray.kill() terminates all worker actors."""
|
||||
executor = RayExecutorV2(vllm_config=create_vllm_config(tensor_parallel_size=2))
|
||||
assert executor.rpc_broadcast_mq is not None
|
||||
assert len(executor.response_mqs) == executor.world_size
|
||||
|
||||
actors = [h.actor for h in executor.ray_worker_handles]
|
||||
executor.shutdown()
|
||||
|
||||
for actor in actors:
|
||||
with pytest.raises(ray.exceptions.RayActorError):
|
||||
ray.get(actor.wait_for_init.remote(), timeout=5)
|
||||
|
||||
assert executor.rpc_broadcast_mq is None
|
||||
assert len(executor.response_mqs) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"executor",
|
||||
[create_vllm_config(tensor_parallel_size=2)],
|
||||
indirect=True,
|
||||
)
|
||||
def test_ray_v2_run_refs_stored_for_monitoring(executor):
|
||||
"""Validate worker handles store run_ref for monitoring."""
|
||||
for handle in executor.ray_worker_handles:
|
||||
assert handle.run_ref is not None
|
||||
ready, _ = ray.wait([handle.run_ref], timeout=0)
|
||||
assert len(ready) == 0, "run_ref should be pending"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size, pp_size", [(2, 1), (2, 2)])
|
||||
def test_ray_v2_single_node_generation(tp_size, pp_size):
|
||||
"""End-to-end LLM generation with RayExecutorV2."""
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
tensor_parallel_size=tp_size,
|
||||
pipeline_parallel_size=pp_size,
|
||||
distributed_executor_backend="ray",
|
||||
enforce_eager=True,
|
||||
max_model_len=256,
|
||||
gpu_memory_utilization=0.3,
|
||||
)
|
||||
try:
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
outputs = llm.generate(prompts)
|
||||
|
||||
assert len(outputs) == len(prompts)
|
||||
for output in outputs:
|
||||
assert len(output.outputs) > 0
|
||||
assert len(output.outputs[0].text) > 0
|
||||
finally:
|
||||
llm.llm_engine.model_executor.shutdown()
|
||||
del llm
|
||||
gc.collect()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bundle_indices, expected_bundle_ids, create_placement_group",
|
||||
[("2,3", [2, 3], 4), ("3,2", [3, 2], 4)],
|
||||
indirect=["create_placement_group"],
|
||||
)
|
||||
def test_ray_v2_bundle_indices_env(
|
||||
bundle_indices, expected_bundle_ids, create_placement_group, monkeypatch
|
||||
):
|
||||
"""Validate explicit VLLM_RAY_BUNDLE_INDICES bundle placement."""
|
||||
monkeypatch.setenv("VLLM_RAY_BUNDLE_INDICES", bundle_indices)
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=2,
|
||||
placement_group=create_placement_group,
|
||||
)
|
||||
executor = RayExecutorV2(vllm_config=vllm_config)
|
||||
try:
|
||||
actual = [
|
||||
h.bundle_id_idx
|
||||
for h in sorted(executor.ray_worker_handles, key=lambda h: h.rank)
|
||||
]
|
||||
assert actual == expected_bundle_ids
|
||||
assert_executor(executor, tp_size=2, pp_size=1)
|
||||
finally:
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bundle_indices, expected_error, create_placement_group",
|
||||
[
|
||||
("1,1", "cannot have duplicate values,", 4),
|
||||
("0,1,2", "must have the same size", 4),
|
||||
],
|
||||
indirect=["create_placement_group"],
|
||||
)
|
||||
def test_ray_v2_invalid_bundle_indices(
|
||||
bundle_indices, expected_error, create_placement_group, monkeypatch
|
||||
):
|
||||
"""Validate invalid bundle indices are rejected."""
|
||||
monkeypatch.setenv("VLLM_RAY_BUNDLE_INDICES", bundle_indices)
|
||||
vllm_config = create_vllm_config(
|
||||
tensor_parallel_size=2, placement_group=create_placement_group
|
||||
)
|
||||
with pytest.raises(AssertionError, match=expected_error):
|
||||
RayExecutorV2(vllm_config=vllm_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size, pp_size", [(2, 1), (2, 2)])
|
||||
def test_ray_v2_single_node_generation_with_pg(tp_size, pp_size):
|
||||
"""E2E LLM generation with a user-provided placement group."""
|
||||
ensure_ray_initialized()
|
||||
bundles = [{"GPU": 1, "CPU": 1} for _ in range(tp_size * pp_size)]
|
||||
pg = ray.util.placement_group(bundles, strategy="PACK")
|
||||
ray.get(pg.ready())
|
||||
|
||||
try:
|
||||
with patch.object(ray.util, "get_current_placement_group", return_value=pg):
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
tensor_parallel_size=tp_size,
|
||||
pipeline_parallel_size=pp_size,
|
||||
distributed_executor_backend="ray",
|
||||
enforce_eager=True,
|
||||
max_model_len=256,
|
||||
gpu_memory_utilization=0.3,
|
||||
)
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
outputs = llm.generate(prompts)
|
||||
|
||||
assert len(outputs) == len(prompts)
|
||||
for output in outputs:
|
||||
assert len(output.outputs) > 0
|
||||
assert len(output.outputs[0].text) > 0
|
||||
finally:
|
||||
llm.llm_engine.model_executor.shutdown()
|
||||
del llm
|
||||
gc.collect()
|
||||
@@ -1,304 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found, rocm_aiter_ops
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce # noqa
|
||||
from vllm.distributed.parallel_state import get_dp_group, get_tp_group, graph_capture
|
||||
from vllm.envs import disable_envs_cache
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import (
|
||||
assert_rocm_custom_allreduce_backend_state,
|
||||
ensure_model_parallel_initialized,
|
||||
init_test_distributed_environment,
|
||||
multi_gpu_test,
|
||||
multi_process_parallel,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm-only AITER custom allreduce tests",
|
||||
)
|
||||
|
||||
test_cases = [
|
||||
((2, 7168), torch.float16),
|
||||
((2, 7168), torch.bfloat16),
|
||||
((128, 8192), torch.float16),
|
||||
((128, 8192), torch.bfloat16),
|
||||
]
|
||||
|
||||
|
||||
def _configure_aiter_custom_ar_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", "NONE")
|
||||
disable_envs_cache()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
|
||||
def _assert_aiter_handles_input(inp: torch.Tensor) -> None:
|
||||
aiter_ar_comm = get_tp_group().device_communicator.aiter_ar_comm
|
||||
assert aiter_ar_comm is not None
|
||||
assert aiter_ar_comm.should_custom_ar(inp), (
|
||||
f"AITER CustomAllreduce does not support input shape {inp.shape}."
|
||||
)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def graph_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
) -> None:
|
||||
with monkeypatch.context() as m:
|
||||
_configure_aiter_custom_ar_env(m)
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
assert_rocm_custom_allreduce_backend_state(True, "NONE")
|
||||
group = get_tp_group().device_group
|
||||
|
||||
# A small all_reduce for warmup.
|
||||
# this is needed because device communicators might be created lazily
|
||||
# (e.g. NCCL). This will ensure that the communicator is initialized
|
||||
# before any communication happens, so that this group can be used for
|
||||
# graph capture immediately.
|
||||
data = torch.zeros(1)
|
||||
data = data.to(device=device)
|
||||
dist.all_reduce(data, group=group)
|
||||
torch.accelerator.synchronize()
|
||||
del data
|
||||
|
||||
for shape, dtype in test_cases:
|
||||
with graph_capture(device=device) as graph_capture_context:
|
||||
inp = torch.ones(shape, dtype=dtype, device=device)
|
||||
_assert_aiter_handles_input(inp)
|
||||
expected = inp * tp_size
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
|
||||
out = tensor_model_parallel_all_reduce(inp)
|
||||
|
||||
graph.replay()
|
||||
torch.testing.assert_close(out, expected)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def eager_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
) -> None:
|
||||
with monkeypatch.context() as m:
|
||||
_configure_aiter_custom_ar_env(m)
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
assert_rocm_custom_allreduce_backend_state(True, "NONE")
|
||||
|
||||
for shape, dtype in test_cases:
|
||||
inp = torch.ones(shape, dtype=dtype, device=device)
|
||||
_assert_aiter_handles_input(inp)
|
||||
expected = inp * tp_size
|
||||
out = tensor_model_parallel_all_reduce(inp)
|
||||
torch.testing.assert_close(out, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_aiter_found(), reason="AITER is not installed")
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("tp_size", [2])
|
||||
@pytest.mark.parametrize("pipeline_parallel_size", [1])
|
||||
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
|
||||
def test_rocm_aiter_custom_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pipeline_parallel_size,
|
||||
test_target,
|
||||
):
|
||||
multi_process_parallel(monkeypatch, tp_size, pipeline_parallel_size, test_target)
|
||||
|
||||
|
||||
def _get_aiter_ag_rs_comm():
|
||||
device_communicator = get_dp_group().device_communicator
|
||||
assert device_communicator.use_aiter_ag_rs, (
|
||||
"AITER custom AG/RS was not enabled on the DP group."
|
||||
)
|
||||
aiter_comm = device_communicator.aiter_ar_comm
|
||||
assert aiter_comm is not None, "AITER custom AG/RS was not initialized."
|
||||
assert not aiter_comm.disabled, "AITER custom AG/RS is disabled."
|
||||
return aiter_comm
|
||||
|
||||
|
||||
def _assert_aiter_handles_ag(aiter_comm, inp: torch.Tensor) -> None:
|
||||
assert aiter_comm.should_custom_ag(inp), (
|
||||
f"AITER custom all-gather does not support input shape {inp.shape}."
|
||||
)
|
||||
|
||||
|
||||
def _assert_aiter_handles_rs(aiter_comm, inp: torch.Tensor) -> None:
|
||||
assert aiter_comm.should_custom_rs(inp, dim=0), (
|
||||
f"AITER custom reduce-scatter does not support input shape {inp.shape}."
|
||||
)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def eager_ag_rs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
data_parallel_size,
|
||||
data_parallel_master_port,
|
||||
) -> None:
|
||||
with monkeypatch.context() as m:
|
||||
_configure_aiter_custom_ar_env(m)
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
data_parallel_size=data_parallel_size,
|
||||
data_parallel_master_port=data_parallel_master_port,
|
||||
)
|
||||
|
||||
dp_group = get_dp_group()
|
||||
group = dp_group.device_group
|
||||
dp_world = dp_group.world_size
|
||||
aiter_comm = _get_aiter_ag_rs_comm()
|
||||
|
||||
for shape, dtype in test_cases:
|
||||
num_tokens, hidden = shape
|
||||
|
||||
# all-gather: each rank contributes (num_tokens, hidden).
|
||||
inp = torch.ones(shape, dtype=dtype, device=device) * (rank + 1)
|
||||
_assert_aiter_handles_ag(aiter_comm, inp)
|
||||
expected = torch.empty(
|
||||
(num_tokens * dp_world, hidden), dtype=dtype, device=device
|
||||
)
|
||||
dist.all_gather_into_tensor(expected, inp, group=group)
|
||||
out = aiter_comm.custom_all_gather(inp, dim=0)
|
||||
assert out is not None
|
||||
torch.testing.assert_close(out, expected)
|
||||
|
||||
# reduce-scatter: each rank contributes (num_tokens * dp, hidden).
|
||||
rs_in = torch.ones(
|
||||
(num_tokens * dp_world, hidden), dtype=dtype, device=device
|
||||
) * (rank + 1)
|
||||
_assert_aiter_handles_rs(aiter_comm, rs_in)
|
||||
rs_expected = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
|
||||
dist.reduce_scatter_tensor(rs_expected, rs_in, group=group)
|
||||
rs_out = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
|
||||
aiter_comm.custom_reduce_scatter(rs_in, rs_out, dim=0)
|
||||
torch.testing.assert_close(rs_out, rs_expected)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def graph_ag_rs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
data_parallel_size,
|
||||
data_parallel_master_port,
|
||||
) -> None:
|
||||
with monkeypatch.context() as m:
|
||||
_configure_aiter_custom_ar_env(m)
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
data_parallel_size=data_parallel_size,
|
||||
data_parallel_master_port=data_parallel_master_port,
|
||||
)
|
||||
|
||||
dp_group = get_dp_group()
|
||||
group = dp_group.device_group
|
||||
dp_world = dp_group.world_size
|
||||
aiter_comm = _get_aiter_ag_rs_comm()
|
||||
|
||||
# Warmup so DP comms is initialized before graph capture
|
||||
data = torch.zeros(1, device=device)
|
||||
dist.all_reduce(data, group=group)
|
||||
torch.accelerator.synchronize()
|
||||
del data
|
||||
|
||||
for shape, dtype in test_cases:
|
||||
num_tokens, hidden = shape
|
||||
|
||||
# all-gather under graph capture.
|
||||
inp = torch.ones(shape, dtype=dtype, device=device) * (rank + 1)
|
||||
_assert_aiter_handles_ag(aiter_comm, inp)
|
||||
ag_expected = torch.empty(
|
||||
(num_tokens * dp_world, hidden), dtype=dtype, device=device
|
||||
)
|
||||
dist.all_gather_into_tensor(ag_expected, inp, group=group)
|
||||
with graph_capture(device=device) as graph_capture_context:
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
|
||||
ag_out = aiter_comm.custom_all_gather(inp, dim=0)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(ag_out, ag_expected)
|
||||
|
||||
# reduce-scatter under graph capture.
|
||||
rs_in = torch.ones(
|
||||
(num_tokens * dp_world, hidden), dtype=dtype, device=device
|
||||
) * (rank + 1)
|
||||
_assert_aiter_handles_rs(aiter_comm, rs_in)
|
||||
rs_expected = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
|
||||
dist.reduce_scatter_tensor(rs_expected, rs_in, group=group)
|
||||
rs_out = torch.empty((num_tokens, hidden), dtype=dtype, device=device)
|
||||
with graph_capture(device=device) as graph_capture_context:
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=graph_capture_context.stream):
|
||||
aiter_comm.custom_reduce_scatter(rs_in, rs_out, dim=0)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(rs_out, rs_expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_aiter_found(), reason="AITER is not installed")
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("tp_size", [1])
|
||||
@pytest.mark.parametrize("pipeline_parallel_size", [1])
|
||||
@pytest.mark.parametrize("data_parallel_size", [2])
|
||||
@pytest.mark.parametrize("test_target", [eager_ag_rs, graph_ag_rs])
|
||||
def test_rocm_aiter_custom_ag_rs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pipeline_parallel_size,
|
||||
data_parallel_size,
|
||||
test_target,
|
||||
):
|
||||
multi_process_parallel(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
pipeline_parallel_size,
|
||||
test_target,
|
||||
data_parallel_size=data_parallel_size,
|
||||
)
|
||||
@@ -1,884 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import contextlib
|
||||
import importlib
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import queue
|
||||
import traceback
|
||||
from functools import lru_cache
|
||||
from types import SimpleNamespace
|
||||
from typing import Literal
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
from ..utils import multi_gpu_test
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm-only quick-reduce tests",
|
||||
)
|
||||
|
||||
MB = 1024 * 1024
|
||||
WORLD_SIZE = 2
|
||||
QUANT_LEVELS = ["FP", "INT8", "INT6", "INT4"]
|
||||
|
||||
|
||||
def _log(message: str) -> None:
|
||||
print(f"[rocm_quick_reduce] {message}", flush=True)
|
||||
|
||||
|
||||
def _reload_envs():
|
||||
return importlib.reload(envs)
|
||||
|
||||
|
||||
def _make_quick_allreduce(
|
||||
*,
|
||||
disabled: bool = False,
|
||||
world_size: int = 2,
|
||||
quant_level: str = "FP",
|
||||
use_fp16_kernels: bool = False,
|
||||
qr_max_size: int = 64 * MB,
|
||||
):
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import (
|
||||
QuickAllReduce,
|
||||
QuickReduceRegime,
|
||||
)
|
||||
|
||||
qar = QuickAllReduce.__new__(QuickAllReduce)
|
||||
qar.disabled = disabled
|
||||
qar.world_size = world_size
|
||||
qar.use_fp16_kernels = use_fp16_kernels
|
||||
qar.qr_quant_level = QuickReduceRegime[quant_level]
|
||||
qar.qr_max_size = qr_max_size
|
||||
qar.qr_min_size = None
|
||||
qar.qr_quantization_min_size = None
|
||||
return qar
|
||||
|
||||
|
||||
def _quick_allreduce_worker(
|
||||
rank: int,
|
||||
port: int,
|
||||
quant_level: str,
|
||||
dtype_name: str,
|
||||
cast_bf16: bool,
|
||||
):
|
||||
os.environ["VLLM_ROCM_QUICK_REDUCE_QUANTIZATION"] = quant_level
|
||||
os.environ["VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16"] = "1" if cast_bf16 else "0"
|
||||
_log(
|
||||
f"worker start: rank={rank} quant={quant_level} "
|
||||
f"dtype={dtype_name} cast_bf16={cast_bf16}"
|
||||
)
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
dist.init_process_group(
|
||||
backend="gloo",
|
||||
init_method=f"tcp://127.0.0.1:{port}",
|
||||
rank=rank,
|
||||
world_size=WORLD_SIZE,
|
||||
)
|
||||
|
||||
qar = None
|
||||
try:
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import (
|
||||
QuickAllReduce,
|
||||
)
|
||||
|
||||
qar = QuickAllReduce(group=dist.GroupMember.WORLD, device=rank)
|
||||
assert not qar.disabled
|
||||
|
||||
num_elements = 8 * MB if dtype_name == "float16" else 4 * MB
|
||||
|
||||
dtype = getattr(torch, dtype_name)
|
||||
inp = torch.ones(num_elements, dtype=dtype, device=device)
|
||||
|
||||
assert qar.should_quick_allreduce(inp)
|
||||
if cast_bf16:
|
||||
assert qar.use_fp16_kernels
|
||||
|
||||
out = qar.quick_all_reduce(inp)
|
||||
assert torch.allclose(out, inp * WORLD_SIZE, atol=2.5, rtol=0.1)
|
||||
_log(
|
||||
f"worker complete: rank={rank} quant={quant_level} "
|
||||
f"dtype={dtype_name} num_elements={num_elements} "
|
||||
f"use_fp16_kernels={qar.use_fp16_kernels}"
|
||||
)
|
||||
finally:
|
||||
if qar is not None:
|
||||
qar.close()
|
||||
if dist.is_initialized():
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
def _run_two_gpu_quick_allreduce_test(
|
||||
*,
|
||||
quant_level: str,
|
||||
dtype_name: str,
|
||||
cast_bf16: bool,
|
||||
):
|
||||
_log(
|
||||
f"launch 2-GPU case: quant={quant_level} "
|
||||
f"dtype={dtype_name} cast_bf16={cast_bf16}"
|
||||
)
|
||||
ctx = mp.get_context("spawn")
|
||||
port = get_open_port()
|
||||
procs = []
|
||||
|
||||
for rank in range(WORLD_SIZE):
|
||||
proc = ctx.Process(
|
||||
target=_quick_allreduce_worker,
|
||||
args=(rank, port, quant_level, dtype_name, cast_bf16),
|
||||
)
|
||||
proc.start()
|
||||
procs.append(proc)
|
||||
|
||||
for proc in procs:
|
||||
proc.join(timeout=60)
|
||||
assert proc.exitcode == 0, f"worker exited with code {proc.exitcode}"
|
||||
_log(
|
||||
f"finished 2-GPU case: quant={quant_level} "
|
||||
f"dtype={dtype_name} cast_bf16={cast_bf16}"
|
||||
)
|
||||
|
||||
|
||||
CUDAGRAPH_WORLD_SIZE = 2
|
||||
CUDAGRAPH_ROUNDS = 10
|
||||
CUDAGRAPH_NUM_ELEMENTS = 1 << 21 # 2M fp16 = 4 MB, above the quick-reduce thresholds
|
||||
|
||||
|
||||
def _quick_allreduce_cudagraph_worker(
|
||||
rank: int,
|
||||
world_size: int,
|
||||
port: int,
|
||||
quant_level: str,
|
||||
):
|
||||
# FP keeps the all-reduce bit-exact for small fp16 integers, so every
|
||||
# replayed round can be checked exactly.
|
||||
os.environ["VLLM_ROCM_QUICK_REDUCE_QUANTIZATION"] = quant_level
|
||||
os.environ["VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16"] = "0"
|
||||
_log(f"cudagraph worker start: rank={rank} quant={quant_level}")
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
dist.init_process_group(
|
||||
backend="gloo",
|
||||
init_method=f"tcp://127.0.0.1:{port}",
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
)
|
||||
|
||||
qar = None
|
||||
try:
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import (
|
||||
QuickAllReduce,
|
||||
)
|
||||
|
||||
qar = QuickAllReduce(group=dist.GroupMember.WORLD, device=rank)
|
||||
assert not qar.disabled
|
||||
|
||||
N = CUDAGRAPH_NUM_ELEMENTS
|
||||
inp = torch.empty(N, dtype=torch.float16, device=device)
|
||||
out = torch.empty(N, dtype=torch.float16, device=device)
|
||||
assert qar.should_quick_allreduce(inp)
|
||||
|
||||
# Every rank contributes the same value v in a round, so the true
|
||||
# cross-rank all-reduce sum is simply world_size * v.
|
||||
def expected(v):
|
||||
return float(world_size * v)
|
||||
|
||||
if rank == 0:
|
||||
print(
|
||||
f"[repro] world_size={world_size} elems={N} regime={quant_level} fp16",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Warmup, then capture a graph with EXACTLY ONE quick-reduce (isolated
|
||||
# qr, so it is the sole writer of its flag slot -- the condition that
|
||||
# triggers the stale-flag bug).
|
||||
inp.fill_(1.0)
|
||||
qar.quick_all_reduce(inp, out=out)
|
||||
torch.accelerator.synchronize()
|
||||
dist.barrier()
|
||||
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
qar.quick_all_reduce(inp, out=out)
|
||||
torch.accelerator.synchronize()
|
||||
dist.barrier()
|
||||
|
||||
for v in range(CUDAGRAPH_ROUNDS):
|
||||
inp.fill_(float(v)) # in-place: same value on every rank
|
||||
dist.barrier()
|
||||
g.replay()
|
||||
torch.accelerator.synchronize()
|
||||
dist.barrier()
|
||||
got = out.float()
|
||||
expect = expected(v)
|
||||
if rank == 0:
|
||||
print(f"round {v}: got={got[:10]}, expected={expect}", flush=True)
|
||||
mismatch = ~torch.isclose(got, torch.full_like(got, expect))
|
||||
num_mismatch = int(mismatch.sum().item())
|
||||
assert num_mismatch == 0, (
|
||||
f"rank={rank} round={v} expected={expect} "
|
||||
f"mismatched {num_mismatch}/{got.numel()} elements; "
|
||||
f"unique wrong values={torch.unique(got[mismatch])[:8].tolist()}"
|
||||
)
|
||||
_log(f"cudagraph worker complete: rank={rank} rounds={CUDAGRAPH_ROUNDS}")
|
||||
finally:
|
||||
if qar is not None:
|
||||
qar.close()
|
||||
if dist.is_initialized():
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
def _run_cudagraph_replay_test(*, world_size: int, quant_level: str):
|
||||
_log(f"launch {world_size}-GPU cudagraph replay case: quant={quant_level}")
|
||||
ctx = mp.get_context("spawn")
|
||||
port = get_open_port()
|
||||
procs = []
|
||||
|
||||
for rank in range(world_size):
|
||||
proc = ctx.Process(
|
||||
target=_quick_allreduce_cudagraph_worker,
|
||||
args=(rank, world_size, port, quant_level),
|
||||
)
|
||||
proc.start()
|
||||
procs.append(proc)
|
||||
|
||||
for proc in procs:
|
||||
proc.join(timeout=120)
|
||||
assert proc.exitcode == 0, f"worker exited with code {proc.exitcode}"
|
||||
_log(f"finished {world_size}-GPU cudagraph replay case: quant={quant_level}")
|
||||
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct"
|
||||
E2E_PREFILL_TOKENS = 1024
|
||||
E2E_MAX_MODEL_LEN = 1536
|
||||
E2E_GPU_MEMORY_UTILIZATION = 0.3
|
||||
E2E_KV_CACHE_MEMORY_BYTES = 2 << 30
|
||||
|
||||
_BACKGROUND_LINE = (
|
||||
"Background filler: this archived operations memo repeats a routine status "
|
||||
"line so the distributed test uses a realistically long prefill."
|
||||
)
|
||||
_BACKGROUND_BLOCK = " ".join([_BACKGROUND_LINE] * 48)
|
||||
|
||||
|
||||
def _build_prompt(*, fact_block: str, question: str) -> str:
|
||||
return (
|
||||
"Read the archived operations memo below. Most of the memo is filler. "
|
||||
"Use only the fact block near the end when answering.\n"
|
||||
f"{_BACKGROUND_BLOCK}\n"
|
||||
"Fact block:\n"
|
||||
f"{fact_block}\n"
|
||||
f"Question: {question}\n"
|
||||
"Answer in one short sentence."
|
||||
)
|
||||
|
||||
|
||||
E2E_PROMPTS = [
|
||||
_build_prompt(
|
||||
fact_block=(
|
||||
"- Festival city: Oslo\n- Mascot animal: otter\n- Welcome drink: tea"
|
||||
),
|
||||
question="Which city hosts the festival, and what animal is the mascot?",
|
||||
),
|
||||
_build_prompt(
|
||||
fact_block=(
|
||||
"- Meeting day: Tuesday\n"
|
||||
"- Planned snack: apricot cake\n"
|
||||
"- Backup room: Cedar"
|
||||
),
|
||||
question="What day is the meeting, and what snack is planned?",
|
||||
),
|
||||
]
|
||||
RECORDED_RESPONSE_TEXTS = (
|
||||
" The city hosting the festival is Oslo, and the mascot is an otter.",
|
||||
" The meeting is on Tuesday and the snack planned is apricot cake.",
|
||||
)
|
||||
REQUIRED_WORDS = (("oslo", "otter"), ("tuesday", "apricot"))
|
||||
|
||||
|
||||
def _log_prompt_summaries() -> None:
|
||||
for i, prompt in enumerate(E2E_PROMPTS):
|
||||
prompt_lines = prompt.splitlines()
|
||||
fact_block = [line for line in prompt_lines if line.startswith("- ")]
|
||||
fact_summary = "; ".join(line.removeprefix("- ") for line in fact_block)
|
||||
_log(f"prompt {i} facts: {fact_summary}")
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _get_model_path() -> str:
|
||||
try:
|
||||
path = hf_api().snapshot_download(repo_id=MODEL_NAME, local_files_only=True)
|
||||
_log(f"using cached model snapshot: {path}")
|
||||
return path
|
||||
except Exception:
|
||||
path = hf_api().snapshot_download(repo_id=MODEL_NAME)
|
||||
_log(f"downloaded model snapshot: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def _get_hidden_size(model_config) -> int:
|
||||
hidden_size = getattr(model_config, "hidden_size", None)
|
||||
if hidden_size is None and hasattr(model_config, "text_config"):
|
||||
hidden_size = getattr(model_config.text_config, "hidden_size", None)
|
||||
assert isinstance(hidden_size, int)
|
||||
return hidden_size
|
||||
|
||||
|
||||
def _check_tp_allreduce_uses_quick_reduce(
|
||||
self,
|
||||
num_tokens: int,
|
||||
dtype_name: str = "float16",
|
||||
) -> dict[str, int | bool]:
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
|
||||
assert self.device is not None
|
||||
qr_comm = get_tp_group().device_communicator.qr_comm
|
||||
assert qr_comm is not None
|
||||
assert not qr_comm.disabled
|
||||
|
||||
hidden_size = _get_hidden_size(self.model_runner.model.config)
|
||||
dtype = getattr(torch, dtype_name)
|
||||
sample = torch.full(
|
||||
(num_tokens, hidden_size),
|
||||
fill_value=float(self.rank + 1),
|
||||
dtype=dtype,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
assert qr_comm.should_quick_allreduce(sample)
|
||||
|
||||
expected = sample.clone()
|
||||
reduced = tensor_model_parallel_all_reduce(sample)
|
||||
dist.all_reduce(expected, group=get_tp_group().device_group)
|
||||
torch.testing.assert_close(reduced, expected, atol=2.5, rtol=0.1)
|
||||
|
||||
stats = {
|
||||
"rank": self.rank,
|
||||
"hidden_size": hidden_size,
|
||||
"num_tokens": num_tokens,
|
||||
"use_fp16_kernels": qr_comm.use_fp16_kernels,
|
||||
}
|
||||
_log(
|
||||
"worker quick-reduce check: "
|
||||
f"rank={self.rank} hidden_size={hidden_size} "
|
||||
f"num_tokens={num_tokens} use_fp16_kernels={qr_comm.use_fp16_kernels}"
|
||||
)
|
||||
return stats
|
||||
|
||||
|
||||
def _check_quick_reduce_disabled(self) -> int:
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
|
||||
qr_comm = get_tp_group().device_communicator.qr_comm
|
||||
assert qr_comm is not None
|
||||
assert qr_comm.disabled
|
||||
_log(f"worker confirmed quick reduce is disabled: rank={self.rank}")
|
||||
return self.rank
|
||||
|
||||
|
||||
def _collect_generations(outputs) -> list[tuple[tuple[int, ...], str]]:
|
||||
return [
|
||||
(tuple(output.outputs[0].token_ids), output.outputs[0].text)
|
||||
for output in outputs
|
||||
]
|
||||
|
||||
|
||||
def _shutdown_llm(llm: LLM | None) -> None:
|
||||
if llm is None:
|
||||
cleanup_dist_env_and_memory()
|
||||
return
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
llm.llm_engine.engine_core.shutdown()
|
||||
|
||||
del llm
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
def _log_generations(
|
||||
label: str,
|
||||
generations: list[tuple[tuple[int, ...], str]],
|
||||
) -> None:
|
||||
for i, (token_ids, text) in enumerate(generations):
|
||||
_log(f"{label} prompt {i} token ids: {list(token_ids)}")
|
||||
_log(f"{label} prompt {i} text: {text!r}")
|
||||
|
||||
|
||||
def _assert_required_words(
|
||||
label: str,
|
||||
generations: list[tuple[tuple[int, ...], str]],
|
||||
) -> None:
|
||||
for i, (_, text) in enumerate(generations):
|
||||
lowered = text.lower()
|
||||
missing = [word for word in REQUIRED_WORDS[i] if word not in lowered]
|
||||
assert not missing, (
|
||||
f"{label} prompt {i} is missing required words {missing}. "
|
||||
f"Observed text: {text!r}"
|
||||
)
|
||||
|
||||
|
||||
def _collect_soft_mismatches(
|
||||
baseline_generations: list[tuple[tuple[int, ...], str]],
|
||||
quick_reduce_generations: list[tuple[tuple[int, ...], str]],
|
||||
) -> list[str]:
|
||||
mismatches = []
|
||||
|
||||
for i, (_, text) in enumerate(baseline_generations):
|
||||
expected = RECORDED_RESPONSE_TEXTS[i]
|
||||
if text != expected:
|
||||
mismatches.append(
|
||||
f"baseline prompt {i} drifted from the recorded response.\n"
|
||||
f"expected={expected!r}\nactual={text!r}"
|
||||
)
|
||||
|
||||
for i, (_, text) in enumerate(quick_reduce_generations):
|
||||
expected = RECORDED_RESPONSE_TEXTS[i]
|
||||
if text != expected:
|
||||
mismatches.append(
|
||||
f"quick-reduce prompt {i} drifted from the recorded response.\n"
|
||||
f"expected={expected!r}\nactual={text!r}"
|
||||
)
|
||||
|
||||
for i, ((_, baseline_text), (_, quick_reduce_text)) in enumerate(
|
||||
zip(baseline_generations, quick_reduce_generations)
|
||||
):
|
||||
if baseline_text != quick_reduce_text:
|
||||
mismatches.append(
|
||||
f"baseline and quick-reduce responses differ for prompt {i}.\n"
|
||||
f"baseline={baseline_text!r}\nquick_reduce={quick_reduce_text!r}"
|
||||
)
|
||||
|
||||
return mismatches
|
||||
|
||||
|
||||
def _run_generation(
|
||||
*,
|
||||
backend: Literal["mp", "ray"],
|
||||
quant_mode: str,
|
||||
expect_quick_reduce: bool,
|
||||
) -> list[tuple[tuple[int, ...], str]]:
|
||||
llm = None
|
||||
monkeypatch = pytest.MonkeyPatch()
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
m.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", quant_mode)
|
||||
model_path = _get_model_path()
|
||||
_log(
|
||||
f"starting generation: backend={backend} quant={quant_mode} "
|
||||
f"gpu_memory_utilization={E2E_GPU_MEMORY_UTILIZATION} "
|
||||
f"kv_cache_bytes={E2E_KV_CACHE_MEMORY_BYTES} model={model_path}"
|
||||
)
|
||||
|
||||
try:
|
||||
llm = LLM(
|
||||
model=model_path,
|
||||
tokenizer=model_path,
|
||||
tensor_parallel_size=2,
|
||||
distributed_executor_backend=backend,
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
max_model_len=E2E_MAX_MODEL_LEN,
|
||||
max_num_seqs=len(E2E_PROMPTS),
|
||||
gpu_memory_utilization=E2E_GPU_MEMORY_UTILIZATION,
|
||||
kv_cache_memory_bytes=E2E_KV_CACHE_MEMORY_BYTES,
|
||||
seed=0,
|
||||
)
|
||||
|
||||
if not expect_quick_reduce:
|
||||
assert llm.collective_rpc(_check_quick_reduce_disabled) == [0, 1]
|
||||
|
||||
if expect_quick_reduce:
|
||||
worker_stats = llm.collective_rpc(
|
||||
_check_tp_allreduce_uses_quick_reduce,
|
||||
args=(E2E_PREFILL_TOKENS,),
|
||||
)
|
||||
assert [stat["rank"] for stat in worker_stats] == [0, 1]
|
||||
worker_summary = "; ".join(
|
||||
"rank={rank} hidden_size={hidden_size} num_tokens={num_tokens} "
|
||||
"use_fp16_kernels={use_fp16_kernels}".format(**stat)
|
||||
for stat in worker_stats
|
||||
)
|
||||
_log(f"{backend} quick-reduce worker checks: {worker_summary}")
|
||||
|
||||
outputs = llm.generate(
|
||||
E2E_PROMPTS,
|
||||
SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=20,
|
||||
stop=["\nAnswer:", " Answer:"],
|
||||
),
|
||||
use_tqdm=False,
|
||||
)
|
||||
generations = _collect_generations(outputs)
|
||||
assert all(text.strip() for _, text in generations)
|
||||
_log_generations(f"{backend} {quant_mode}", generations)
|
||||
return generations
|
||||
finally:
|
||||
_shutdown_llm(llm)
|
||||
|
||||
|
||||
def _run_quick_reduce_llm_e2e_in_subprocess(
|
||||
*,
|
||||
backend: Literal["mp", "ray"],
|
||||
) -> str | None:
|
||||
_log(f"running LLM e2e: backend={backend}")
|
||||
_log_prompt_summaries()
|
||||
baseline_outputs = _run_generation(
|
||||
backend=backend,
|
||||
quant_mode="NONE",
|
||||
expect_quick_reduce=False,
|
||||
)
|
||||
quick_reduce_outputs = _run_generation(
|
||||
backend=backend,
|
||||
quant_mode="FP",
|
||||
expect_quick_reduce=True,
|
||||
)
|
||||
|
||||
_assert_required_words("baseline", baseline_outputs)
|
||||
_assert_required_words("quick-reduce", quick_reduce_outputs)
|
||||
|
||||
mismatches = _collect_soft_mismatches(baseline_outputs, quick_reduce_outputs)
|
||||
if mismatches:
|
||||
details = "\n\n".join(mismatches)
|
||||
_log(f"soft response mismatch:\n{details}")
|
||||
return details
|
||||
|
||||
_log(f"LLM e2e backend={backend} matched the recorded responses exactly")
|
||||
return None
|
||||
|
||||
|
||||
def _quick_reduce_llm_e2e_worker(
|
||||
result_queue: mp.Queue,
|
||||
backend: Literal["mp", "ray"],
|
||||
) -> None:
|
||||
try:
|
||||
xfail_reason = _run_quick_reduce_llm_e2e_in_subprocess(backend=backend)
|
||||
except Exception:
|
||||
result_queue.put({"status": "error", "reason": traceback.format_exc()})
|
||||
raise
|
||||
else:
|
||||
if xfail_reason is not None:
|
||||
result_queue.put({"status": "xfail", "reason": xfail_reason})
|
||||
else:
|
||||
result_queue.put({"status": "ok"})
|
||||
|
||||
|
||||
def run_quick_reduce_llm_e2e(
|
||||
*,
|
||||
backend: Literal["mp", "ray"],
|
||||
) -> None:
|
||||
ctx = mp.get_context("spawn")
|
||||
result_queue = ctx.Queue()
|
||||
proc = ctx.Process(
|
||||
target=_quick_reduce_llm_e2e_worker,
|
||||
args=(result_queue, backend),
|
||||
)
|
||||
proc.start()
|
||||
proc.join(timeout=600)
|
||||
|
||||
try:
|
||||
result = result_queue.get(timeout=5)
|
||||
except queue.Empty as exc:
|
||||
if proc.exitcode != 0:
|
||||
raise AssertionError(
|
||||
f"quick-reduce llm e2e subprocess failed for backend={backend} "
|
||||
f"with exit code {proc.exitcode} and produced no result"
|
||||
) from exc
|
||||
raise AssertionError(
|
||||
f"quick-reduce llm e2e subprocess produced no result for backend={backend}"
|
||||
) from exc
|
||||
|
||||
if result["status"] == "xfail":
|
||||
pytest.xfail(result["reason"])
|
||||
if result["status"] == "error":
|
||||
raise AssertionError(
|
||||
f"quick-reduce llm e2e subprocess failed for backend={backend}:\n"
|
||||
f"{result['reason']}"
|
||||
)
|
||||
|
||||
assert proc.exitcode == 0, (
|
||||
f"quick-reduce llm e2e subprocess failed for backend={backend} "
|
||||
f"with exit code {proc.exitcode}"
|
||||
)
|
||||
|
||||
|
||||
def test_quick_reduce_regime_values():
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import QuickReduceRegime
|
||||
|
||||
assert QuickReduceRegime.FP.value == 0
|
||||
assert QuickReduceRegime.INT8.value == 1
|
||||
assert QuickReduceRegime.INT6.value == 2
|
||||
assert QuickReduceRegime.INT4.value == 3
|
||||
assert QuickReduceRegime.INT3.value == 4
|
||||
assert QuickReduceRegime.NONE.value == 5
|
||||
|
||||
|
||||
def test_quick_reduce_regime_names():
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import QuickReduceRegime
|
||||
|
||||
assert set(QuickReduceRegime.__members__) == {
|
||||
"FP",
|
||||
"INT8",
|
||||
"INT6",
|
||||
"INT4",
|
||||
"INT3",
|
||||
"NONE",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quant_level", QUANT_LEVELS + ["NONE"])
|
||||
def test_quick_reduce_quantization_env_var(monkeypatch, quant_level):
|
||||
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", quant_level)
|
||||
|
||||
reloaded_envs = _reload_envs()
|
||||
assert quant_level == reloaded_envs.VLLM_ROCM_QUICK_REDUCE_QUANTIZATION
|
||||
|
||||
|
||||
def test_quick_reduce_quantization_default(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", raising=False)
|
||||
|
||||
reloaded_envs = _reload_envs()
|
||||
assert reloaded_envs.VLLM_ROCM_QUICK_REDUCE_QUANTIZATION == "NONE"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cast_bf16", [True, False])
|
||||
def test_quick_reduce_cast_bf16_to_fp16_env_var(monkeypatch, cast_bf16):
|
||||
monkeypatch.setenv(
|
||||
"VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", "1" if cast_bf16 else "0"
|
||||
)
|
||||
|
||||
reloaded_envs = _reload_envs()
|
||||
assert reloaded_envs.VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16 is cast_bf16
|
||||
|
||||
|
||||
def test_quick_reduce_cast_bf16_to_fp16_default(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", raising=False)
|
||||
|
||||
reloaded_envs = _reload_envs()
|
||||
assert reloaded_envs.VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16 is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_mb", [128, 512, 2048, None])
|
||||
def test_quick_reduce_max_size_env_var(monkeypatch, max_mb):
|
||||
if max_mb is None:
|
||||
monkeypatch.delenv("VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", str(max_mb))
|
||||
|
||||
reloaded_envs = _reload_envs()
|
||||
assert max_mb == reloaded_envs.VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB
|
||||
|
||||
|
||||
def test_quick_reduce_max_size_default(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", raising=False)
|
||||
|
||||
reloaded_envs = _reload_envs()
|
||||
assert reloaded_envs.VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("gcn_arch_name", "expected"),
|
||||
[
|
||||
("gfx942", True),
|
||||
("gfx950", True),
|
||||
("gfx90a", False),
|
||||
("", False),
|
||||
],
|
||||
)
|
||||
def test_quick_allreduce_rocm_arch_available(gcn_arch_name, expected):
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import QuickAllReduce
|
||||
|
||||
qar = QuickAllReduce.__new__(QuickAllReduce)
|
||||
qar.disabled = True
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.device_communicators.quick_all_reduce.current_platform."
|
||||
"is_rocm",
|
||||
return_value=True,
|
||||
),
|
||||
patch(
|
||||
"torch.cuda.get_device_properties",
|
||||
return_value=SimpleNamespace(gcnArchName=gcn_arch_name),
|
||||
),
|
||||
):
|
||||
assert qar._rocm_arch_available() is expected
|
||||
|
||||
|
||||
def test_quick_allreduce_rocm_arch_available_handles_probe_failure():
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import QuickAllReduce
|
||||
|
||||
qar = QuickAllReduce.__new__(QuickAllReduce)
|
||||
qar.disabled = True
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.device_communicators.quick_all_reduce.current_platform."
|
||||
"is_rocm",
|
||||
return_value=True,
|
||||
),
|
||||
patch("torch.cuda.get_device_properties", side_effect=RuntimeError),
|
||||
):
|
||||
assert qar._rocm_arch_available() is False
|
||||
|
||||
|
||||
def test_quick_allreduce_rejects_disabled():
|
||||
qar = _make_quick_allreduce(disabled=True)
|
||||
|
||||
inp = torch.zeros(1024, dtype=torch.float16)
|
||||
assert qar.should_quick_allreduce(inp) is False
|
||||
|
||||
|
||||
def test_quick_allreduce_rejects_unsupported_dtype():
|
||||
qar = _make_quick_allreduce()
|
||||
|
||||
inp = torch.zeros(1024 * 1024, dtype=torch.float32)
|
||||
assert qar.should_quick_allreduce(inp) is False
|
||||
|
||||
|
||||
def test_quick_allreduce_rejects_non_aligned_input():
|
||||
qar = _make_quick_allreduce()
|
||||
|
||||
inp = torch.zeros(5, dtype=torch.float16)
|
||||
assert qar.should_quick_allreduce(inp) is False
|
||||
|
||||
|
||||
def test_quick_allreduce_rejects_non_contiguous_input():
|
||||
qar = _make_quick_allreduce()
|
||||
|
||||
inp = torch.zeros((1024, 1024), dtype=torch.float16)[:, ::2]
|
||||
assert qar.should_quick_allreduce(inp) is False
|
||||
|
||||
|
||||
def test_quick_allreduce_rejects_input_smaller_than_threshold():
|
||||
qar = _make_quick_allreduce()
|
||||
|
||||
inp = torch.zeros((MB // 2) - 8, dtype=torch.float16)
|
||||
assert qar.should_quick_allreduce(inp) is False
|
||||
|
||||
|
||||
def test_quick_allreduce_accepts_input_at_threshold():
|
||||
qar = _make_quick_allreduce()
|
||||
|
||||
inp = torch.zeros(MB // 2, dtype=torch.float16)
|
||||
assert qar.should_quick_allreduce(inp) is True
|
||||
|
||||
|
||||
def test_quick_allreduce_rejects_input_larger_than_max_size():
|
||||
qar = _make_quick_allreduce(qr_max_size=1 * MB)
|
||||
|
||||
inp = torch.zeros(MB, dtype=torch.float16)
|
||||
assert qar.should_quick_allreduce(inp) is False
|
||||
|
||||
|
||||
def test_quick_allreduce_bf16_uses_fp16_threshold_when_cast_enabled():
|
||||
inp = torch.zeros(MB // 2, dtype=torch.bfloat16)
|
||||
|
||||
without_cast = _make_quick_allreduce(use_fp16_kernels=False)
|
||||
with_cast = _make_quick_allreduce(use_fp16_kernels=True)
|
||||
|
||||
assert without_cast.should_quick_allreduce(inp) is False
|
||||
assert with_cast.should_quick_allreduce(inp) is True
|
||||
|
||||
|
||||
def test_quick_allreduce_supported_world_sizes():
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import QuickAllReduce
|
||||
|
||||
assert QuickAllReduce._SUPPORTED_WORLD_SIZES == [2, 4, 8]
|
||||
|
||||
|
||||
def test_quick_allreduce_supported_dtypes():
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import QuickAllReduce
|
||||
|
||||
assert [torch.float16, torch.bfloat16] == QuickAllReduce._SUPPORTED_DTYPES
|
||||
|
||||
|
||||
def test_quick_allreduce_min_size_table():
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import QuickAllReduce
|
||||
|
||||
for dtype in [torch.float16, torch.bfloat16]:
|
||||
for world_size in QuickAllReduce._SUPPORTED_WORLD_SIZES:
|
||||
min_sizes = QuickAllReduce._QR_MIN_SIZE[(dtype, world_size)]
|
||||
assert len(min_sizes) == 5
|
||||
assert all(size > 0 for size in min_sizes)
|
||||
|
||||
|
||||
def test_qr_max_size():
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
max_size = ops.qr_max_size()
|
||||
assert isinstance(max_size, int)
|
||||
assert max_size > 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.device_count() < WORLD_SIZE,
|
||||
reason="requires 2 ROCm GPUs",
|
||||
)
|
||||
@pytest.mark.parametrize("quant_level", QUANT_LEVELS)
|
||||
def test_quick_allreduce_two_gpu_correctness(quant_level):
|
||||
_log(f"two-GPU correctness case: quant={quant_level}")
|
||||
_run_two_gpu_quick_allreduce_test(
|
||||
quant_level=quant_level,
|
||||
dtype_name="float16",
|
||||
cast_bf16=False,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=CUDAGRAPH_WORLD_SIZE)
|
||||
def test_quick_allreduce_cudagraph_replay():
|
||||
# Regression test for the stale flag_color bug: a quick-reduce captured in a
|
||||
# CUDA graph must return correct results on every replay, not the data from
|
||||
# a previous round.
|
||||
_log("cudagraph replay case")
|
||||
_run_cudagraph_replay_test(
|
||||
world_size=CUDAGRAPH_WORLD_SIZE,
|
||||
quant_level="FP",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.device_count() < WORLD_SIZE,
|
||||
reason="requires 2 ROCm GPUs",
|
||||
)
|
||||
def test_quick_allreduce_bf16_cast_mode():
|
||||
_log("BF16 cast case")
|
||||
_run_two_gpu_quick_allreduce_test(
|
||||
quant_level="FP",
|
||||
dtype_name="bfloat16",
|
||||
cast_bf16=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.device_count() < WORLD_SIZE,
|
||||
reason="requires 2 ROCm GPUs",
|
||||
)
|
||||
def test_quick_allreduce_llm_e2e():
|
||||
_log("LLM e2e case: backend=mp")
|
||||
run_quick_reduce_llm_e2e(backend="mp")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.device_count() < WORLD_SIZE,
|
||||
reason="requires 2 ROCm GPUs",
|
||||
)
|
||||
def test_quick_allreduce_llm_e2e_ray():
|
||||
_log("LLM e2e case: backend=ray")
|
||||
run_quick_reduce_llm_e2e(backend="ray")
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,140 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import queue
|
||||
import random
|
||||
import typing
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
|
||||
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tp_group,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.engine.llm_engine import LLMEngine
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
torch.manual_seed(42)
|
||||
random.seed(44)
|
||||
|
||||
test_size_elements = 1024 * 1024
|
||||
|
||||
|
||||
def symm_mem_allreduce_worker(local_rank: int, world_size: int, q: mp.Queue):
|
||||
monkeypatch = pytest.MonkeyPatch()
|
||||
config = VllmConfig(parallel_config=ParallelConfig(tensor_parallel_size=world_size))
|
||||
|
||||
with monkeypatch.context() as m, set_current_vllm_config(config):
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
dtype = torch.bfloat16
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": "12345",
|
||||
}
|
||||
)
|
||||
|
||||
init_distributed_environment()
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
|
||||
cuda_communicator = typing.cast(
|
||||
CudaCommunicator, get_tp_group().device_communicator
|
||||
)
|
||||
symm_mem_comm = cuda_communicator.symm_mem_comm
|
||||
if symm_mem_comm is None or symm_mem_comm.disabled:
|
||||
# can't use skip under multiprocessing
|
||||
q.put("SymmMemCommunicator is not available or disabled.")
|
||||
return
|
||||
|
||||
inp_direct_symm_mem = torch.randint(
|
||||
1, 23, (test_size_elements,), dtype=dtype, device=device
|
||||
)
|
||||
if not symm_mem_comm.should_use_symm_mem(inp_direct_symm_mem):
|
||||
# can't use skip under multiprocessing
|
||||
q.put("SymmMemCommunicator isn't used for this world and input size.")
|
||||
return
|
||||
|
||||
original_inp_direct_symm_mem = inp_direct_symm_mem.clone()
|
||||
out_direct_symm_mem = symm_mem_comm.all_reduce(inp_direct_symm_mem)
|
||||
assert out_direct_symm_mem is not None
|
||||
|
||||
group = get_tp_group().device_group
|
||||
dist.all_reduce(original_inp_direct_symm_mem, group=group)
|
||||
torch.testing.assert_close(
|
||||
out_direct_symm_mem, original_inp_direct_symm_mem, atol=2.5, rtol=0.1
|
||||
)
|
||||
|
||||
# Test tensor_model_parallel_all_reduce which should use symm_mem
|
||||
inp_tensor_parallel = torch.randint(
|
||||
-23, 1, (test_size_elements,), dtype=dtype, device=device
|
||||
)
|
||||
original_inp_tensor_parallel = inp_tensor_parallel.clone()
|
||||
out_tensor_parallel = tensor_model_parallel_all_reduce(inp_tensor_parallel)
|
||||
dist.all_reduce(original_inp_tensor_parallel, group=group)
|
||||
torch.testing.assert_close(
|
||||
out_tensor_parallel, original_inp_tensor_parallel, atol=2.5, rtol=0.1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="SymmMemAllreduce is only available for CUDA platforms.",
|
||||
)
|
||||
@pytest.mark.parametrize("tp_size", [2])
|
||||
@pytest.mark.parametrize("pipeline_parallel_size", [1])
|
||||
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
|
||||
def test_symm_mem_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch, tp_size, pipeline_parallel_size
|
||||
):
|
||||
world_size = tp_size * pipeline_parallel_size
|
||||
if world_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
q = mp.get_context("spawn").Queue()
|
||||
mp.spawn(symm_mem_allreduce_worker, args=(world_size, q), nprocs=world_size)
|
||||
try:
|
||||
val = q.get(timeout=1)
|
||||
except queue.Empty:
|
||||
val = None
|
||||
finally:
|
||||
cleanup_dist_env_and_memory()
|
||||
if val is not None:
|
||||
pytest.skip(val)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="SymmMemAllreduce is only available for CUDA platforms.",
|
||||
)
|
||||
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
|
||||
def test_dp_with_symm_mem_allreduce(monkeypatch: pytest.MonkeyPatch):
|
||||
world_size = 4
|
||||
if world_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
# Verify that the DataParallel runs without error
|
||||
engine_args = EngineArgs(
|
||||
model="distilbert/distilgpt2",
|
||||
enforce_eager=True,
|
||||
enable_prefix_caching=True,
|
||||
data_parallel_size=2,
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_backend="mp",
|
||||
)
|
||||
LLMEngine.from_engine_args(engine_args)
|
||||
@@ -1,82 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# unit test for `examples/features/torchrun/torchrun_example_offline.py`
|
||||
import os
|
||||
import random
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
# By default, let PyTorch choose the WORLD backend for the current device
|
||||
# type (legacy lazy-init path). When VLLM_DISTRIBUTED_USE_SPLIT_GROUP=1,
|
||||
# use the explicit eager-init pattern required by `split_group` (mixed
|
||||
# cpu:gloo,cuda:nccl backend + device_id binding).
|
||||
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
|
||||
local_rank = int(os.environ["LOCAL_RANK"])
|
||||
torch.accelerator.set_device_index(local_rank)
|
||||
dist.init_process_group(
|
||||
backend="cpu:gloo,cuda:nccl",
|
||||
device_id=torch.device(f"cuda:{local_rank}"),
|
||||
)
|
||||
else:
|
||||
dist.init_process_group()
|
||||
|
||||
# Create prompts
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
|
||||
|
||||
# set different `gpu_memory_utilization` for different ranks,
|
||||
# to test if all ranks agree on the same kv cache configuration.
|
||||
llm = LLM(
|
||||
model="facebook/opt-125m",
|
||||
tensor_parallel_size=2,
|
||||
pipeline_parallel_size=int(os.getenv("PP_SIZE", 1)),
|
||||
distributed_executor_backend="external_launcher",
|
||||
gpu_memory_utilization=random.uniform(0.8, 0.92),
|
||||
seed=0,
|
||||
)
|
||||
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
|
||||
cpu_group = get_world_group().cpu_group
|
||||
|
||||
torch_rank = dist.get_rank(group=cpu_group)
|
||||
|
||||
|
||||
def test_consistent_across_ranks(obj):
|
||||
if torch_rank == 0:
|
||||
dist.broadcast_object_list([obj], src=0, group=cpu_group)
|
||||
else:
|
||||
container = [None]
|
||||
dist.broadcast_object_list(container, src=0, group=cpu_group)
|
||||
assert container[0] == obj
|
||||
|
||||
|
||||
test_consistent_across_ranks(llm.llm_engine.vllm_config.cache_config.num_cpu_blocks)
|
||||
test_consistent_across_ranks(llm.llm_engine.vllm_config.cache_config.num_gpu_blocks)
|
||||
|
||||
# make sure we can access the model parameters from the calling process
|
||||
# of the `LLM` instance.
|
||||
params = list(
|
||||
llm.llm_engine.model_executor.driver_worker.worker.model_runner.model.parameters()
|
||||
)
|
||||
test_consistent_across_ranks(len(params))
|
||||
|
||||
# all ranks should have the same outputs
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
test_consistent_across_ranks(prompt)
|
||||
test_consistent_across_ranks(generated_text)
|
||||
print(f"Rank {torch_rank}, Prompt: {prompt!r}, Generated text: {generated_text!r}")
|
||||
@@ -1,91 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# unit test for `examples/features/torchrun/torchrun_example_offline.py`
|
||||
import os
|
||||
import random
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed.parallel_state import get_tp_group, get_world_group
|
||||
|
||||
# By default, let PyTorch choose the WORLD backend for the current device
|
||||
# type (legacy lazy-init path). When VLLM_DISTRIBUTED_USE_SPLIT_GROUP=1,
|
||||
# use the explicit eager-init pattern required by `split_group` (mixed
|
||||
# cpu:gloo,cuda:nccl backend + device_id binding).
|
||||
if envs.VLLM_DISTRIBUTED_USE_SPLIT_GROUP:
|
||||
local_rank = int(os.environ["LOCAL_RANK"])
|
||||
torch.accelerator.set_device_index(local_rank)
|
||||
dist.init_process_group(
|
||||
backend="cpu:gloo,cuda:nccl",
|
||||
device_id=torch.device(f"cuda:{local_rank}"),
|
||||
)
|
||||
else:
|
||||
dist.init_process_group()
|
||||
|
||||
# Create prompts
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
] * 10
|
||||
dp_size = int(os.getenv("DP_SIZE", "1"))
|
||||
dp_rank = int(os.getenv("DP_RANK", "0"))
|
||||
|
||||
if dp_size > 1:
|
||||
# distribute the prompts across the data parallel ranks
|
||||
prompts = [prompt for idx, prompt in enumerate(prompts) if idx % dp_size == dp_rank]
|
||||
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
|
||||
|
||||
# set different `gpu_memory_utilization` for different ranks,
|
||||
# to test if all ranks agree on the same kv cache configuration.
|
||||
llm = LLM(
|
||||
model="microsoft/Phi-mini-MoE-instruct",
|
||||
tensor_parallel_size=int(os.getenv("TP_SIZE", "1")),
|
||||
pipeline_parallel_size=int(os.getenv("PP_SIZE", "1")),
|
||||
enable_expert_parallel=int(os.getenv("ENABLE_EP", "0")) == 1,
|
||||
distributed_executor_backend="external_launcher",
|
||||
gpu_memory_utilization=random.uniform(0.8, 0.92),
|
||||
seed=0,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=16,
|
||||
)
|
||||
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
|
||||
group = get_world_group() if dp_size == 1 else get_tp_group()
|
||||
cpu_group = group.cpu_group
|
||||
group_rank = dist.get_rank(group=cpu_group)
|
||||
|
||||
|
||||
def test_consistent_across_ranks(obj):
|
||||
if group_rank == 0:
|
||||
dist.broadcast_object_list([obj], src=group.ranks[0], group=cpu_group)
|
||||
else:
|
||||
container = [None]
|
||||
dist.broadcast_object_list(container, src=group.ranks[0], group=cpu_group)
|
||||
assert container[0] == obj
|
||||
|
||||
|
||||
test_consistent_across_ranks(llm.llm_engine.vllm_config.cache_config.num_cpu_blocks)
|
||||
test_consistent_across_ranks(llm.llm_engine.vllm_config.cache_config.num_gpu_blocks)
|
||||
|
||||
# make sure we can access the model parameters from the calling process
|
||||
# of the `LLM` instance.
|
||||
params = list(
|
||||
llm.llm_engine.model_executor.driver_worker.worker.model_runner.model.parameters()
|
||||
)
|
||||
test_consistent_across_ranks(len(params))
|
||||
|
||||
# all ranks should have the same outputs
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
test_consistent_across_ranks(prompt)
|
||||
test_consistent_across_ranks(generated_text)
|
||||
print(f"Rank {group_rank}, Prompt: {prompt!r}, Generated text: {generated_text!r}")
|
||||
@@ -1,141 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import socket
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
from ..utils import multi_gpu_test
|
||||
|
||||
|
||||
@ray.remote
|
||||
class _CUDADeviceCountStatelessTestActor:
|
||||
def get_count(self):
|
||||
return current_platform.device_count()
|
||||
|
||||
def set_cuda_visible_devices(self, cuda_visible_devices: str):
|
||||
update_environment_variables({"CUDA_VISIBLE_DEVICES": cuda_visible_devices})
|
||||
|
||||
def get_cuda_visible_devices(self):
|
||||
return envs.CUDA_VISIBLE_DEVICES
|
||||
|
||||
|
||||
def test_cuda_device_count_stateless():
|
||||
"""Test that cuda_device_count_stateless changes return value if
|
||||
CUDA_VISIBLE_DEVICES is changed."""
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip("Skip for ROCm because Ray uses HIP_VISIBLE_DEVICES.")
|
||||
actor = _CUDADeviceCountStatelessTestActor.options( # type: ignore
|
||||
num_gpus=2
|
||||
).remote()
|
||||
assert len(sorted(ray.get(actor.get_cuda_visible_devices.remote()).split(","))) == 2
|
||||
assert ray.get(actor.get_count.remote()) == 2
|
||||
ray.get(actor.set_cuda_visible_devices.remote("0"))
|
||||
assert ray.get(actor.get_count.remote()) == 1
|
||||
ray.get(actor.set_cuda_visible_devices.remote(""))
|
||||
assert ray.get(actor.get_count.remote()) == 0
|
||||
|
||||
|
||||
def cpu_worker(rank, WORLD_SIZE, port1, port2):
|
||||
pg1 = StatelessProcessGroup.create(
|
||||
host="127.0.0.1", port=port1, rank=rank, world_size=WORLD_SIZE
|
||||
)
|
||||
if rank <= 2:
|
||||
pg2 = StatelessProcessGroup.create(
|
||||
host="127.0.0.1", port=port2, rank=rank, world_size=3
|
||||
)
|
||||
data = torch.tensor([rank])
|
||||
data = pg1.broadcast_obj(data, src=2)
|
||||
assert data.item() == 2
|
||||
if rank <= 2:
|
||||
data = torch.tensor([rank + 1])
|
||||
data = pg2.broadcast_obj(data, src=2)
|
||||
assert data.item() == 3
|
||||
pg2.barrier()
|
||||
pg1.barrier()
|
||||
|
||||
|
||||
def gpu_worker(rank, WORLD_SIZE, port1, port2):
|
||||
torch.accelerator.set_device_index(rank)
|
||||
pg1 = StatelessProcessGroup.create(
|
||||
host="127.0.0.1", port=port1, rank=rank, world_size=WORLD_SIZE
|
||||
)
|
||||
pynccl1 = PyNcclCommunicator(pg1, device=rank)
|
||||
if rank <= 2:
|
||||
pg2 = StatelessProcessGroup.create(
|
||||
host="127.0.0.1", port=port2, rank=rank, world_size=3
|
||||
)
|
||||
pynccl2 = PyNcclCommunicator(pg2, device=rank)
|
||||
data = torch.tensor([rank]).cuda()
|
||||
pynccl1.all_reduce(data)
|
||||
pg1.barrier()
|
||||
torch.accelerator.synchronize()
|
||||
if rank <= 2:
|
||||
pynccl2.all_reduce(data)
|
||||
pg2.barrier()
|
||||
torch.accelerator.synchronize()
|
||||
item = data[0].item()
|
||||
print(f"rank: {rank}, item: {item}")
|
||||
if rank == 3:
|
||||
assert item == 6
|
||||
else:
|
||||
assert item == 18
|
||||
|
||||
|
||||
def broadcast_worker(rank, WORLD_SIZE, port1, port2):
|
||||
pg1 = StatelessProcessGroup.create(
|
||||
host="127.0.0.1", port=port1, rank=rank, world_size=WORLD_SIZE
|
||||
)
|
||||
if rank == 2:
|
||||
pg1.broadcast_obj("secret", src=2)
|
||||
else:
|
||||
obj = pg1.broadcast_obj(None, src=2)
|
||||
assert obj == "secret"
|
||||
pg1.barrier()
|
||||
|
||||
|
||||
def allgather_worker(rank, WORLD_SIZE, port1, port2):
|
||||
pg1 = StatelessProcessGroup.create(
|
||||
host="127.0.0.1", port=port1, rank=rank, world_size=WORLD_SIZE
|
||||
)
|
||||
data = pg1.all_gather_obj(rank)
|
||||
assert data == list(range(WORLD_SIZE))
|
||||
pg1.barrier()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="This test is flaky and prone to hang.")
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
@pytest.mark.parametrize(
|
||||
"worker", [cpu_worker, gpu_worker, broadcast_worker, allgather_worker]
|
||||
)
|
||||
def test_stateless_process_group(worker):
|
||||
port1 = get_open_port()
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("", port1))
|
||||
port2 = get_open_port()
|
||||
WORLD_SIZE = 4
|
||||
from multiprocessing import get_context
|
||||
|
||||
ctx = get_context("fork")
|
||||
processes = []
|
||||
for i in range(WORLD_SIZE):
|
||||
rank = i
|
||||
processes.append(
|
||||
ctx.Process(target=worker, args=(rank, WORLD_SIZE, port1, port2))
|
||||
)
|
||||
for p in processes:
|
||||
p.start()
|
||||
for p in processes:
|
||||
p.join()
|
||||
for p in processes:
|
||||
assert not p.exitcode
|
||||
print("All processes finished.")
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user