[Distributed] Support pre-shared ncclUniqueId rendezvous for weight transfer (#53784)
Signed-off-by: Dharak Kharod <dharak.kharod@cohere.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,465 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the pre-shared ncclUniqueId weight-transfer rendezvous.
|
||||
|
||||
The rendezvous lets a peer that cannot join a TCPStore (e.g. a torch-free JAX
|
||||
trainer) initialize NCCL weight transfer from a unique id minted out of band.
|
||||
|
||||
* Unit tests for the mutually-exclusive rendezvous-mode validation on the NCCL
|
||||
init infos.
|
||||
* Two-GPU integration tests driving the real worker engine against a peer: one
|
||||
via vLLM's own ``PyNcclCommunicator``, one via a genuinely torch-free trainer
|
||||
(cupy buffer + raw ``ncclBroadcast``).
|
||||
|
||||
The shared Ray helpers come from the sibling module (unchanged); the worker task
|
||||
is local so the sibling's TCP test and its worker stay untouched.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
|
||||
from tests.distributed.test_weight_transfer import (
|
||||
_init_ray_for_weight_transfer,
|
||||
_set_ray_assigned_device,
|
||||
create_mock_vllm_config,
|
||||
)
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerInitInfo,
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLWeightTransferInitInfo,
|
||||
)
|
||||
|
||||
# Standard base64 of exactly 128 zero bytes: a well-formed (dummy) ncclUniqueId.
|
||||
VALID_UID_B64 = base64.b64encode(b"\x00" * 128).decode()
|
||||
|
||||
|
||||
class TestNCCLRendezvousValidation:
|
||||
"""Test the xor-of-rendezvous-modes validation on the NCCL init infos."""
|
||||
|
||||
def test_worker_init_info_is_keyword_only(self):
|
||||
# kw_only so a stale positional call fails loudly rather than silently
|
||||
# swapping fields once the rendezvous fields became optional.
|
||||
with pytest.raises(TypeError):
|
||||
NCCLWeightTransferInitInfo(1, 3, "127.0.0.1", 12345) # type: ignore[call-arg]
|
||||
|
||||
def test_trainer_init_info_accepts_positional_fields(self):
|
||||
# NCCLTrainerInitInfo keeps positional subclass fields; only `rank`
|
||||
# (from TrainerInitInfo) is keyword-only.
|
||||
trainer_positional = NCCLTrainerInitInfo("127.0.0.1", 12345, 3, rank=0)
|
||||
trainer_keyword = NCCLTrainerInitInfo(
|
||||
master_address="127.0.0.1",
|
||||
master_port=12345,
|
||||
world_size=3,
|
||||
rank=0,
|
||||
)
|
||||
assert trainer_positional == trainer_keyword
|
||||
|
||||
def test_tcp_mode_valid(self):
|
||||
info = NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
master_address="127.0.0.1",
|
||||
master_port=12345,
|
||||
)
|
||||
assert info.master_address == "127.0.0.1"
|
||||
assert info.nccl_unique_id_b64 is None
|
||||
|
||||
def test_uid_mode_valid(self):
|
||||
info = NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
nccl_unique_id_b64=VALID_UID_B64,
|
||||
)
|
||||
assert info.nccl_unique_id_b64 == VALID_UID_B64
|
||||
assert info.master_address is None
|
||||
assert info.master_port is None
|
||||
assert VALID_UID_B64 not in repr(info)
|
||||
|
||||
def test_both_modes_raises(self):
|
||||
with pytest.raises(ValueError, match="not both"):
|
||||
NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
master_address="127.0.0.1",
|
||||
master_port=12345,
|
||||
nccl_unique_id_b64=VALID_UID_B64,
|
||||
)
|
||||
|
||||
def test_neither_mode_raises(self):
|
||||
with pytest.raises(ValueError, match="need nccl_unique_id_b64"):
|
||||
NCCLWeightTransferInitInfo(rank_offset=1, world_size=3)
|
||||
|
||||
def test_half_specified_tcp_raises(self):
|
||||
with pytest.raises(ValueError, match="master_port"):
|
||||
NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
master_address="127.0.0.1",
|
||||
)
|
||||
|
||||
def test_bad_base64_uid_raises(self):
|
||||
with pytest.raises(ValueError, match="not valid standard base64"):
|
||||
NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
nccl_unique_id_b64="!!!!",
|
||||
)
|
||||
|
||||
def test_wrong_length_uid_raises(self):
|
||||
short_uid = base64.b64encode(b"\x00" * 64).decode()
|
||||
with pytest.raises(ValueError, match="128"):
|
||||
NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
nccl_unique_id_b64=short_uid,
|
||||
)
|
||||
|
||||
def test_parse_init_info_uid_valid(self):
|
||||
engine = NCCLWeightTransferEngine(
|
||||
WeightTransferConfig(backend="nccl"),
|
||||
create_mock_vllm_config(),
|
||||
torch.device("cuda"),
|
||||
MagicMock(spec=torch.nn.Module),
|
||||
)
|
||||
init_info = engine.parse_init_info(
|
||||
{
|
||||
"nccl_unique_id_b64": VALID_UID_B64,
|
||||
"rank_offset": 1,
|
||||
"world_size": 3,
|
||||
}
|
||||
)
|
||||
assert isinstance(init_info, NCCLWeightTransferInitInfo)
|
||||
assert init_info.nccl_unique_id_b64 == VALID_UID_B64
|
||||
assert init_info.master_address is None
|
||||
|
||||
def test_parse_init_info_both_modes_raises(self):
|
||||
engine = NCCLWeightTransferEngine(
|
||||
WeightTransferConfig(backend="nccl"),
|
||||
create_mock_vllm_config(),
|
||||
torch.device("cuda"),
|
||||
MagicMock(spec=torch.nn.Module),
|
||||
)
|
||||
with pytest.raises(ValueError, match="not both"):
|
||||
engine.parse_init_info(
|
||||
{
|
||||
"master_address": "127.0.0.1",
|
||||
"master_port": 12345,
|
||||
"nccl_unique_id_b64": VALID_UID_B64,
|
||||
"rank_offset": 1,
|
||||
"world_size": 3,
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("unique_id_bytes", "rank", "world_size", "match"),
|
||||
[
|
||||
(b"\x00" * 64, 0, 2, "128-byte"),
|
||||
(b"\x00" * 128, -1, 2, "out of range"),
|
||||
(b"\x00" * 128, 2, 2, "out of range"),
|
||||
],
|
||||
)
|
||||
def test_from_unique_id_bytes_validation(
|
||||
self, unique_id_bytes, rank, world_size, match
|
||||
):
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
with pytest.raises(ValueError, match=match):
|
||||
PyNcclCommunicator.from_unique_id_bytes(
|
||||
unique_id_bytes,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
device=0,
|
||||
)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def inference_receive_tensor(
|
||||
init_info_dict: dict,
|
||||
tensor_shape: list[int],
|
||||
tensor_dtype: str,
|
||||
) -> dict:
|
||||
"""Worker that joins via the given init dict and records the tensor it gets.
|
||||
|
||||
Local to this module (rather than the sibling's TCP-only worker) so it can
|
||||
take a UID init payload -- the same dict boundary an external trainer POSTs.
|
||||
"""
|
||||
import contextlib
|
||||
|
||||
import torch
|
||||
|
||||
_set_ray_assigned_device()
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLWeightTransferInitInfo,
|
||||
NCCLWeightTransferUpdateInfo,
|
||||
)
|
||||
|
||||
class Recorder(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.received = []
|
||||
|
||||
def load_weights(self, weights):
|
||||
for name, tensor in weights:
|
||||
self.received.append((name, tensor.clone()))
|
||||
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
vllm_config = MagicMock()
|
||||
parallel_config = MagicMock(spec=ParallelConfig)
|
||||
parallel_config.rank = 0
|
||||
parallel_config.world_size = 1
|
||||
parallel_config.data_parallel_rank = 0
|
||||
parallel_config.data_parallel_index = 0
|
||||
vllm_config.parallel_config = parallel_config
|
||||
vllm_config.model_config = MagicMock()
|
||||
|
||||
recorder = Recorder()
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, vllm_config, torch.device("cuda"), recorder
|
||||
)
|
||||
# Transport-only test: bypass the set_current_vllm_config context that
|
||||
# receive_weights enters, since vllm_config here is a mock.
|
||||
import vllm.config as _vllm_config_mod
|
||||
|
||||
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
|
||||
|
||||
try:
|
||||
engine.init_transfer_engine(NCCLWeightTransferInitInfo(**init_info_dict))
|
||||
engine.receive_weights(
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=["test.weight"],
|
||||
dtype_names=[tensor_dtype],
|
||||
shapes=[tensor_shape],
|
||||
)
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
received_shape = None
|
||||
received_sum = None
|
||||
if len(recorder.received) == 1:
|
||||
_name, tensor = recorder.received[0]
|
||||
received_shape = list(tensor.shape)
|
||||
received_sum = tensor.sum().item()
|
||||
|
||||
return {
|
||||
"success": received_shape == tensor_shape
|
||||
and abs(received_sum - torch.tensor(tensor_shape).prod().item()) < 0.01,
|
||||
"received_shape": received_shape,
|
||||
"received_sum": received_sum,
|
||||
}
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def trainer_broadcast_tensor_uid(
|
||||
uid_b64: str,
|
||||
world_size: int,
|
||||
tensor_shape: list[int],
|
||||
tensor_dtype: str,
|
||||
) -> dict:
|
||||
"""Drive rank 0 via vLLM's own communicator from a pre-shared unique id.
|
||||
|
||||
A real torch-free trainer would mint the id and drive rank 0 itself; here we
|
||||
reuse the real send protocol by injecting the communicator into a sender
|
||||
engine, exercising the worker-side UID path end to end.
|
||||
"""
|
||||
import pybase64 as base64
|
||||
import torch
|
||||
|
||||
_set_ray_assigned_device()
|
||||
|
||||
from vllm.distributed.weight_transfer import ModuleSource
|
||||
from vllm.distributed.weight_transfer.nccl_common import uid_init_process_group
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerWeightTransferEngine,
|
||||
)
|
||||
|
||||
class NoopClient:
|
||||
def init_weight_transfer_engine(self, init_info):
|
||||
pass
|
||||
|
||||
def start_weight_update(self):
|
||||
pass
|
||||
|
||||
def update_weights(self, update_info):
|
||||
pass
|
||||
|
||||
def finish_weight_update(self, weight_version=None):
|
||||
pass
|
||||
|
||||
dtype = getattr(torch, tensor_dtype)
|
||||
model = torch.nn.Module()
|
||||
test_module = torch.nn.Module()
|
||||
test_module.register_parameter(
|
||||
"weight",
|
||||
torch.nn.Parameter(
|
||||
torch.ones(tensor_shape, dtype=dtype, device="cuda"), requires_grad=False
|
||||
),
|
||||
)
|
||||
model.add_module("test", test_module)
|
||||
|
||||
engine = NCCLTrainerWeightTransferEngine(
|
||||
client=NoopClient(),
|
||||
source=ModuleSource(model),
|
||||
is_sender=True,
|
||||
packed=False,
|
||||
)
|
||||
engine.model_update_group = uid_init_process_group(
|
||||
base64.b64decode(uid_b64),
|
||||
rank=0,
|
||||
world_size=world_size,
|
||||
device=torch.accelerator.current_device_index(),
|
||||
)
|
||||
try:
|
||||
engine.send_weights()
|
||||
torch.accelerator.synchronize()
|
||||
return {"success": True}
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def trainer_broadcast_tensor_torch_free(
|
||||
uid_b64: str,
|
||||
world_size: int,
|
||||
tensor_shape: list[int],
|
||||
tensor_dtype: str,
|
||||
) -> dict:
|
||||
"""Trainer peer that uses no torch and no PyNcclCommunicator (a JAX stand-in).
|
||||
|
||||
Drives NCCL through the raw ctypes wrapper with a cupy send buffer, so it
|
||||
exercises interop with a foreign NCCL peer rather than vLLM against itself.
|
||||
``PyNcclCommunicator.__init__`` runs a one-element warm-up all_reduce right
|
||||
after ``ncclCommInitRank``; that is a collective, so this peer must issue the
|
||||
matching all_reduce before the broadcast or both ranks deadlock.
|
||||
"""
|
||||
import cupy as cp
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
|
||||
device = _set_ray_assigned_device()
|
||||
cp.cuda.Device(device.index).use()
|
||||
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||
NCCLLibrary,
|
||||
buffer_type,
|
||||
cudaStream_t,
|
||||
ncclDataTypeEnum,
|
||||
ncclRedOpTypeEnum,
|
||||
)
|
||||
|
||||
assert tensor_dtype == "float32", "harness only wires float32"
|
||||
# Fill on the host and copy up (a plain H2D memcpy) so we neither use torch
|
||||
# nor make cupy JIT-compile a fill kernel (which needs NVRTC). All-ones lets
|
||||
# the worker verify by sum == numel.
|
||||
sendbuff = cp.asarray(np.ones(tensor_shape, dtype=np.float32))
|
||||
warmup = cp.asarray(np.zeros(1, dtype=np.float32))
|
||||
numel = int(sendbuff.size)
|
||||
# The copies above run on the default stream; drain them before the NCCL
|
||||
# collectives read the buffers on our own stream.
|
||||
cp.cuda.Device(device.index).synchronize()
|
||||
stream = cp.cuda.Stream()
|
||||
|
||||
nccl = NCCLLibrary()
|
||||
comm = nccl.ncclCommInitRank(
|
||||
world_size, nccl.unique_id_from_bytes(base64.b64decode(uid_b64)), 0
|
||||
)
|
||||
try:
|
||||
# Mirror the worker's init-time warm-up all_reduce (rank 0 must join it).
|
||||
nccl.ncclAllReduce(
|
||||
buffer_type(warmup.data.ptr),
|
||||
buffer_type(warmup.data.ptr),
|
||||
1,
|
||||
ncclDataTypeEnum.ncclFloat32,
|
||||
ncclRedOpTypeEnum.ncclSum,
|
||||
comm,
|
||||
cudaStream_t(stream.ptr),
|
||||
)
|
||||
nccl.ncclBroadcast(
|
||||
buffer_type(sendbuff.data.ptr),
|
||||
buffer_type(sendbuff.data.ptr),
|
||||
numel,
|
||||
ncclDataTypeEnum.ncclFloat32,
|
||||
0, # root
|
||||
comm,
|
||||
cudaStream_t(stream.ptr),
|
||||
)
|
||||
stream.synchronize()
|
||||
return {"success": True}
|
||||
finally:
|
||||
# Abort (not the collective destroy) for an uncoordinated teardown.
|
||||
nccl.ncclCommAbort(comm)
|
||||
|
||||
|
||||
def _run_uid_transfer(trainer_task) -> None:
|
||||
"""Mint a unique id, run the worker and the given trainer peer concurrently,
|
||||
and assert the tensor round-trips. Both ranks must enter init together --
|
||||
there is no store barrier on the UID path."""
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
|
||||
|
||||
_init_ray_for_weight_transfer()
|
||||
|
||||
nccl = NCCLLibrary()
|
||||
uid_b64 = base64.b64encode(bytes(nccl.ncclGetUniqueId().internal)).decode()
|
||||
|
||||
world_size = 2 # 1 trainer + 1 inference worker
|
||||
tensor_shape = [100, 100]
|
||||
tensor_dtype = "float32"
|
||||
|
||||
inference_future = inference_receive_tensor.remote(
|
||||
{
|
||||
"rank_offset": 1,
|
||||
"world_size": world_size,
|
||||
"nccl_unique_id_b64": uid_b64,
|
||||
"packed": False,
|
||||
},
|
||||
tensor_shape,
|
||||
tensor_dtype,
|
||||
)
|
||||
trainer_future = trainer_task.remote(
|
||||
uid_b64, world_size, tensor_shape, tensor_dtype
|
||||
)
|
||||
|
||||
try:
|
||||
trainer_result, result = ray.get(
|
||||
[trainer_future, inference_future], timeout=120
|
||||
)
|
||||
finally:
|
||||
ray.shutdown()
|
||||
|
||||
assert trainer_result["success"], "Trainer should complete successfully"
|
||||
assert result["success"], (
|
||||
f"Weight transfer failed. "
|
||||
f"Received shape: {result['received_shape']}, "
|
||||
f"Received sum: {result['received_sum']}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 2,
|
||||
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
|
||||
)
|
||||
def test_nccl_weight_transfer_between_processes_uid():
|
||||
"""Weight transfer over a pre-shared ncclUniqueId (no TCPStore), with both
|
||||
ranks on vLLM's ``PyNcclCommunicator``."""
|
||||
_run_uid_transfer(trainer_broadcast_tensor_uid)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 2,
|
||||
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
|
||||
)
|
||||
def test_nccl_weight_transfer_torch_free_trainer():
|
||||
"""UID weight transfer against a foreign peer that uses no torch and no vLLM
|
||||
communicator, only a cupy buffer and raw ``ncclBroadcast``."""
|
||||
pytest.importorskip("cupy")
|
||||
_run_uid_transfer(trainer_broadcast_tensor_torch_free)
|
||||
@@ -11,6 +11,7 @@ from torch.distributed import ProcessGroup, ReduceOp
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||
NCCL_UNIQUE_ID_BYTES,
|
||||
NCCLLibrary,
|
||||
buffer_type,
|
||||
cudaStream_t,
|
||||
@@ -60,6 +61,9 @@ def register_nccl_symmetric_ops(pynccl_comm):
|
||||
|
||||
|
||||
class PyNcclCommunicator:
|
||||
# None for communicators built via `from_unique_id_bytes` (no process group).
|
||||
group: ProcessGroup | StatelessProcessGroup | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup | StatelessProcessGroup,
|
||||
@@ -128,6 +132,14 @@ class PyNcclCommunicator:
|
||||
self.unique_id.internal[i] = byte
|
||||
else:
|
||||
self.unique_id = group.broadcast_obj(self.unique_id, src=0)
|
||||
self._init_comm(device)
|
||||
|
||||
def _init_comm(self, device: int | str | torch.device) -> None:
|
||||
"""Create the communicator on `device` from the already-resolved
|
||||
`self.unique_id` / `self.rank` / `self.world_size`, then run the
|
||||
one-element warm-up all_reduce. Shared by `__init__` and
|
||||
`from_unique_id_bytes` so the init handshake stays identical on both.
|
||||
"""
|
||||
if isinstance(device, int):
|
||||
device = torch.device(f"cuda:{device}")
|
||||
elif isinstance(device, str):
|
||||
@@ -148,6 +160,62 @@ class PyNcclCommunicator:
|
||||
stream.synchronize()
|
||||
del data
|
||||
|
||||
@classmethod
|
||||
def from_unique_id_bytes(
|
||||
cls,
|
||||
unique_id_bytes: bytes,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
device: int | str | torch.device,
|
||||
library_path: str | None = None,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Build a communicator from pre-shared ``ncclUniqueId`` bytes.
|
||||
|
||||
For peers that cannot join a ``StatelessProcessGroup`` / TCPStore (e.g. a
|
||||
torch-free JAX trainer): every rank passes the same id, minted once via
|
||||
``ncclGetUniqueId`` and shared out of band. There is no barrier, so all
|
||||
ranks must enter init concurrently or ``ncclCommInitRank`` hangs.
|
||||
|
||||
Warm-up handshake: immediately after ``ncclCommInitRank`` this issues a
|
||||
one-element ``all_reduce`` (mirroring ``__init__``). It is a collective,
|
||||
so every peer -- including a foreign, non-vLLM rank -- must issue a
|
||||
matching one-element ``all_reduce`` before any other collective, or all
|
||||
ranks deadlock.
|
||||
"""
|
||||
if len(unique_id_bytes) != NCCL_UNIQUE_ID_BYTES:
|
||||
raise ValueError(
|
||||
f"expected a {NCCL_UNIQUE_ID_BYTES}-byte NCCL unique id, "
|
||||
f"got {len(unique_id_bytes)} bytes"
|
||||
)
|
||||
if not 0 <= rank < world_size:
|
||||
raise ValueError(f"rank {rank} out of range for world_size {world_size}")
|
||||
|
||||
self = cls.__new__(cls)
|
||||
self.rank = rank
|
||||
self.world_size = world_size
|
||||
self.group = None
|
||||
|
||||
if self.world_size == 1 or envs.VLLM_DISABLE_PYNCCL:
|
||||
self.available = False
|
||||
self.disabled = True
|
||||
return self
|
||||
try:
|
||||
self.nccl = NCCLLibrary(library_path)
|
||||
except Exception as e:
|
||||
# Unlike the TCPStore path, silently disabling here leaves the peer
|
||||
# blocked in ncclCommInitRank until timeout. The caller explicitly
|
||||
# asked to join from a unique id, so fail loudly instead.
|
||||
raise RuntimeError(
|
||||
"failed to load the NCCL library for unique-id rendezvous"
|
||||
) from e
|
||||
|
||||
self.available = True
|
||||
self.disabled = False
|
||||
self.nccl_version = self.nccl.ncclGetRawVersion()
|
||||
self.unique_id = self.nccl.unique_id_from_bytes(unique_id_bytes)
|
||||
self._init_comm(device)
|
||||
return self
|
||||
|
||||
def destroy(self):
|
||||
if self.available and not self.disabled:
|
||||
# ncclCommAbort can block until all CUDA graphs that
|
||||
|
||||
@@ -51,6 +51,9 @@ class ncclUniqueId(ctypes.Structure):
|
||||
_fields_ = [("internal", ctypes.c_byte * 128)]
|
||||
|
||||
|
||||
NCCL_UNIQUE_ID_BYTES = ctypes.sizeof(ncclUniqueId)
|
||||
|
||||
|
||||
# Mirror of NCCL's versioned ncclCommProperties_t (nccl_device/core.h,
|
||||
# v2.31.2-1, 144 bytes). ncclCommQueryProperties fills fields gated by the
|
||||
# version the caller declares in props.version, not by props.size, so never
|
||||
|
||||
@@ -8,15 +8,19 @@ The dense (`NCCLWeightTransferEngine`) and sparse
|
||||
sparse engine does not have to subclass the dense one.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
import pybase64 as base64
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||
NCCL_UNIQUE_ID_BYTES,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
@@ -24,21 +28,102 @@ from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Worker-side initialization info for NCCL-based weight transfer backends."""
|
||||
def decode_nccl_unique_id(
|
||||
*,
|
||||
master_address: str | None,
|
||||
master_port: int | None,
|
||||
nccl_unique_id_b64: str | None,
|
||||
ctx: str,
|
||||
) -> bytes | None:
|
||||
"""Validate the rendezvous mode and decode a pre-shared unique id."""
|
||||
has_uid = bool(nccl_unique_id_b64)
|
||||
has_addr = master_address is not None
|
||||
has_port = master_port is not None
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
if has_uid and (has_addr or has_port):
|
||||
raise ValueError(
|
||||
f"{ctx}: pass nccl_unique_id_b64 OR master_address/master_port, not both"
|
||||
)
|
||||
if not has_uid:
|
||||
if not (has_addr and has_port):
|
||||
raise ValueError(
|
||||
f"{ctx}: need nccl_unique_id_b64, or both master_address and "
|
||||
f"master_port (got master_address={master_address!r}, "
|
||||
f"master_port={master_port!r})"
|
||||
)
|
||||
return None
|
||||
try:
|
||||
decoded = base64.b64decode(nccl_unique_id_b64, validate=True)
|
||||
except (ValueError, TypeError) as e:
|
||||
raise ValueError(
|
||||
f"{ctx}: nccl_unique_id_b64 is not valid standard base64 "
|
||||
f"(URL-safe base64 is not accepted)"
|
||||
) from e
|
||||
if len(decoded) != NCCL_UNIQUE_ID_BYTES:
|
||||
raise ValueError(
|
||||
f"{ctx}: nccl_unique_id_b64 must decode to {NCCL_UNIQUE_ID_BYTES} "
|
||||
f"bytes, got {len(decoded)}"
|
||||
)
|
||||
return decoded
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Worker-side initialization info for NCCL-based weight transfer backends.
|
||||
|
||||
Keyword-only (`kw_only`): adding the optional `nccl_unique_id_b64` field
|
||||
means the rendezvous fields can no longer keep a fixed positional slot, so a
|
||||
stale positional call fails loudly instead of silently swapping arguments.
|
||||
|
||||
Provide exactly one rendezvous mode:
|
||||
|
||||
* ``master_address`` + ``master_port`` -- TCPStore / ``StatelessProcessGroup``
|
||||
rendezvous (requires torch on every rank, including the trainer), or
|
||||
* ``nccl_unique_id_b64`` -- standard (RFC 4648, *not* URL-safe) base64 of the
|
||||
128 raw bytes from ``ncclGetUniqueId``, for torch-free trainers (e.g. JAX)
|
||||
that mint the unique id out of band and share it (over HTTP, etc.). Note a
|
||||
JAX peer must use ``base64.b64encode``, not ``urlsafe_b64encode``.
|
||||
|
||||
On the unique-id path all ranks must enter init concurrently (there is no
|
||||
store barrier), and every peer must honor the warm-up handshake: the
|
||||
worker's communicator issues a one-element ``all_reduce`` immediately after
|
||||
``ncclCommInitRank`` (see ``PyNcclCommunicator.from_unique_id_bytes``), so a
|
||||
foreign peer must issue a matching one-element ``all_reduce`` before any
|
||||
other collective or all ranks deadlock.
|
||||
"""
|
||||
|
||||
master_address: str | None = None
|
||||
master_port: int | None = None
|
||||
rank_offset: int
|
||||
world_size: int
|
||||
nccl_unique_id_b64: str | None = field(default=None, repr=False)
|
||||
packed: bool = False
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_ = self.nccl_unique_id_bytes
|
||||
|
||||
@property
|
||||
def nccl_unique_id_bytes(self) -> bytes | None:
|
||||
return decode_nccl_unique_id(
|
||||
master_address=self.master_address,
|
||||
master_port=self.master_port,
|
||||
nccl_unique_id_b64=self.nccl_unique_id_b64,
|
||||
ctx="NCCLWeightTransferInitInfo",
|
||||
)
|
||||
|
||||
|
||||
def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
|
||||
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
|
||||
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
|
||||
only the mode actually in use. Shared by the dense and sparse trainer
|
||||
engines so the two cannot drift."""
|
||||
return {key: value for key, value in asdict(init_info).items() if value is not None}
|
||||
|
||||
|
||||
class NCCLRendezvous(Protocol):
|
||||
"""The three fields `trainer_init` needs to open the trainer endpoint.
|
||||
"""The TCP rendezvous fields `trainer_init` needs.
|
||||
|
||||
Structural so each backend can keep its own trainer init info next to its
|
||||
engine (dense `NCCLTrainerInitInfo`, `SparseNCCLTrainerInitInfo`) without
|
||||
@@ -73,6 +158,27 @@ def stateless_init_process_group(
|
||||
return PyNcclCommunicator(pg, device=device)
|
||||
|
||||
|
||||
def uid_init_process_group(
|
||||
nccl_unique_id_bytes: bytes,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
device,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Join the NCCL group from pre-shared ``ncclUniqueId`` bytes.
|
||||
|
||||
The torch-free rendezvous alternative to `stateless_init_process_group`: no
|
||||
TCPStore, and therefore no barrier -- every rank must enter concurrently.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
return PyNcclCommunicator.from_unique_id_bytes(
|
||||
nccl_unique_id_bytes,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
def worker_init_process_group(
|
||||
init_info: NCCLWeightTransferInitInfo,
|
||||
parallel_config: "ParallelConfig",
|
||||
@@ -80,7 +186,7 @@ def worker_init_process_group(
|
||||
"""Create the trainer<->worker NCCL group on an inference worker.
|
||||
|
||||
Computes a unique rank for this worker across all data-parallel groups and
|
||||
joins the stateless process group with the trainer.
|
||||
joins the trainer via whichever rendezvous mode `init_info` carries.
|
||||
"""
|
||||
# Calculate the global rank in the trainer-worker process group.
|
||||
# Must account for data parallel to get unique ranks across all workers.
|
||||
@@ -93,6 +199,17 @@ def worker_init_process_group(
|
||||
rank = worker_rank + init_info.rank_offset
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
unique_id_bytes = init_info.nccl_unique_id_bytes
|
||||
if unique_id_bytes is not None:
|
||||
return uid_init_process_group(
|
||||
unique_id_bytes,
|
||||
rank,
|
||||
init_info.world_size,
|
||||
device,
|
||||
)
|
||||
# __post_init__ guarantees the TCP pair when there is no unique id.
|
||||
assert init_info.master_address is not None
|
||||
assert init_info.master_port is not None
|
||||
return stateless_init_process_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.distributed.weight_transfer.base import (
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
worker_init_payload,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
@@ -57,6 +58,13 @@ class NCCLTrainerInitInfo(TrainerInitInfo):
|
||||
`world_size` is the full trainer+worker NCCL group size. `rank` (from
|
||||
`TrainerInitInfo`) identifies this trainer process; rank 0 is the sender.
|
||||
|
||||
The trainer joins over a TCPStore rendezvous (`master_address` +
|
||||
`master_port`). Torch-free trainers (e.g. JAX) that cannot join a TCPStore
|
||||
mint an `ncclUniqueId` themselves and drive rank 0 out of band; they ship a
|
||||
`nccl_unique_id_b64` payload straight to the inference workers'
|
||||
`init_weight_transfer_engine` (see `NCCLWeightTransferInitInfo`) rather than
|
||||
going through this engine.
|
||||
|
||||
`packed` / buffer sizes are the transfer's wire params. The trainer
|
||||
propagates them to the worker at `trainer_init` so the two sides cannot
|
||||
disagree. Note this defaults to packed, unlike the worker-side
|
||||
@@ -301,7 +309,8 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni
|
||||
# open the trainer endpoint (rank 0); both sides must rendezvous together.
|
||||
with ThreadPoolExecutor(max_workers=1) as exe:
|
||||
future = exe.submit(
|
||||
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
|
||||
engine.client.init_weight_transfer_engine,
|
||||
worker_init_payload(worker_init_info),
|
||||
)
|
||||
engine.model_update_group = open_trainer_endpoint(init_info)
|
||||
future.result() # surface any inference-side init error
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.distributed.weight_transfer.base import (
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
worker_init_payload,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
@@ -256,7 +257,8 @@ class SparseNCCLTrainerWeightTransferEngine(
|
||||
# open the trainer endpoint (rank 0); both sides must rendezvous together.
|
||||
with ThreadPoolExecutor(max_workers=1) as exe:
|
||||
future = exe.submit(
|
||||
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
|
||||
engine.client.init_weight_transfer_engine,
|
||||
worker_init_payload(worker_init_info),
|
||||
)
|
||||
# Open the trainer endpoint as NCCL rank 0 on the current device
|
||||
# (the init info satisfies the helper's rendezvous protocol).
|
||||
|
||||
Reference in New Issue
Block a user