[Distributed] Support pre-shared ncclUniqueId rendezvous for weight transfer (#53784)

Signed-off-by: Dharak Kharod <dharak.kharod@cohere.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
dharak-cohere
2026-09-01 14:55:57 +08:00
committed by GitHub
co-authored by Cursor
parent fa99a6fea6
commit d0e695a91b
6 changed files with 674 additions and 10 deletions
@@ -0,0 +1,465 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the pre-shared ncclUniqueId weight-transfer rendezvous.
The rendezvous lets a peer that cannot join a TCPStore (e.g. a torch-free JAX
trainer) initialize NCCL weight transfer from a unique id minted out of band.
* Unit tests for the mutually-exclusive rendezvous-mode validation on the NCCL
init infos.
* Two-GPU integration tests driving the real worker engine against a peer: one
via vLLM's own ``PyNcclCommunicator``, one via a genuinely torch-free trainer
(cupy buffer + raw ``ncclBroadcast``).
The shared Ray helpers come from the sibling module (unchanged); the worker task
is local so the sibling's TCP test and its worker stay untouched.
"""
from unittest.mock import MagicMock
import pybase64 as base64
import pytest
import ray
import torch
from tests.distributed.test_weight_transfer import (
_init_ray_for_weight_transfer,
_set_ray_assigned_device,
create_mock_vllm_config,
)
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerInitInfo,
NCCLWeightTransferEngine,
NCCLWeightTransferInitInfo,
)
# Standard base64 of exactly 128 zero bytes: a well-formed (dummy) ncclUniqueId.
VALID_UID_B64 = base64.b64encode(b"\x00" * 128).decode()
class TestNCCLRendezvousValidation:
"""Test the xor-of-rendezvous-modes validation on the NCCL init infos."""
def test_worker_init_info_is_keyword_only(self):
# kw_only so a stale positional call fails loudly rather than silently
# swapping fields once the rendezvous fields became optional.
with pytest.raises(TypeError):
NCCLWeightTransferInitInfo(1, 3, "127.0.0.1", 12345) # type: ignore[call-arg]
def test_trainer_init_info_accepts_positional_fields(self):
# NCCLTrainerInitInfo keeps positional subclass fields; only `rank`
# (from TrainerInitInfo) is keyword-only.
trainer_positional = NCCLTrainerInitInfo("127.0.0.1", 12345, 3, rank=0)
trainer_keyword = NCCLTrainerInitInfo(
master_address="127.0.0.1",
master_port=12345,
world_size=3,
rank=0,
)
assert trainer_positional == trainer_keyword
def test_tcp_mode_valid(self):
info = NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
master_address="127.0.0.1",
master_port=12345,
)
assert info.master_address == "127.0.0.1"
assert info.nccl_unique_id_b64 is None
def test_uid_mode_valid(self):
info = NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
nccl_unique_id_b64=VALID_UID_B64,
)
assert info.nccl_unique_id_b64 == VALID_UID_B64
assert info.master_address is None
assert info.master_port is None
assert VALID_UID_B64 not in repr(info)
def test_both_modes_raises(self):
with pytest.raises(ValueError, match="not both"):
NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
master_address="127.0.0.1",
master_port=12345,
nccl_unique_id_b64=VALID_UID_B64,
)
def test_neither_mode_raises(self):
with pytest.raises(ValueError, match="need nccl_unique_id_b64"):
NCCLWeightTransferInitInfo(rank_offset=1, world_size=3)
def test_half_specified_tcp_raises(self):
with pytest.raises(ValueError, match="master_port"):
NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
master_address="127.0.0.1",
)
def test_bad_base64_uid_raises(self):
with pytest.raises(ValueError, match="not valid standard base64"):
NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
nccl_unique_id_b64="!!!!",
)
def test_wrong_length_uid_raises(self):
short_uid = base64.b64encode(b"\x00" * 64).decode()
with pytest.raises(ValueError, match="128"):
NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
nccl_unique_id_b64=short_uid,
)
def test_parse_init_info_uid_valid(self):
engine = NCCLWeightTransferEngine(
WeightTransferConfig(backend="nccl"),
create_mock_vllm_config(),
torch.device("cuda"),
MagicMock(spec=torch.nn.Module),
)
init_info = engine.parse_init_info(
{
"nccl_unique_id_b64": VALID_UID_B64,
"rank_offset": 1,
"world_size": 3,
}
)
assert isinstance(init_info, NCCLWeightTransferInitInfo)
assert init_info.nccl_unique_id_b64 == VALID_UID_B64
assert init_info.master_address is None
def test_parse_init_info_both_modes_raises(self):
engine = NCCLWeightTransferEngine(
WeightTransferConfig(backend="nccl"),
create_mock_vllm_config(),
torch.device("cuda"),
MagicMock(spec=torch.nn.Module),
)
with pytest.raises(ValueError, match="not both"):
engine.parse_init_info(
{
"master_address": "127.0.0.1",
"master_port": 12345,
"nccl_unique_id_b64": VALID_UID_B64,
"rank_offset": 1,
"world_size": 3,
}
)
@pytest.mark.parametrize(
("unique_id_bytes", "rank", "world_size", "match"),
[
(b"\x00" * 64, 0, 2, "128-byte"),
(b"\x00" * 128, -1, 2, "out of range"),
(b"\x00" * 128, 2, 2, "out of range"),
],
)
def test_from_unique_id_bytes_validation(
self, unique_id_bytes, rank, world_size, match
):
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
with pytest.raises(ValueError, match=match):
PyNcclCommunicator.from_unique_id_bytes(
unique_id_bytes,
rank=rank,
world_size=world_size,
device=0,
)
@ray.remote(num_gpus=1)
def inference_receive_tensor(
init_info_dict: dict,
tensor_shape: list[int],
tensor_dtype: str,
) -> dict:
"""Worker that joins via the given init dict and records the tensor it gets.
Local to this module (rather than the sibling's TCP-only worker) so it can
take a UID init payload -- the same dict boundary an external trainer POSTs.
"""
import contextlib
import torch
_set_ray_assigned_device()
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLWeightTransferEngine,
NCCLWeightTransferInitInfo,
NCCLWeightTransferUpdateInfo,
)
class Recorder(torch.nn.Module):
def __init__(self):
super().__init__()
self.received = []
def load_weights(self, weights):
for name, tensor in weights:
self.received.append((name, tensor.clone()))
config = WeightTransferConfig(backend="nccl")
vllm_config = MagicMock()
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
vllm_config.parallel_config = parallel_config
vllm_config.model_config = MagicMock()
recorder = Recorder()
engine = NCCLWeightTransferEngine(
config, vllm_config, torch.device("cuda"), recorder
)
# Transport-only test: bypass the set_current_vllm_config context that
# receive_weights enters, since vllm_config here is a mock.
import vllm.config as _vllm_config_mod
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
try:
engine.init_transfer_engine(NCCLWeightTransferInitInfo(**init_info_dict))
engine.receive_weights(
NCCLWeightTransferUpdateInfo(
names=["test.weight"],
dtype_names=[tensor_dtype],
shapes=[tensor_shape],
)
)
torch.accelerator.synchronize()
received_shape = None
received_sum = None
if len(recorder.received) == 1:
_name, tensor = recorder.received[0]
received_shape = list(tensor.shape)
received_sum = tensor.sum().item()
return {
"success": received_shape == tensor_shape
and abs(received_sum - torch.tensor(tensor_shape).prod().item()) < 0.01,
"received_shape": received_shape,
"received_sum": received_sum,
}
finally:
engine.shutdown()
@ray.remote(num_gpus=1)
def trainer_broadcast_tensor_uid(
uid_b64: str,
world_size: int,
tensor_shape: list[int],
tensor_dtype: str,
) -> dict:
"""Drive rank 0 via vLLM's own communicator from a pre-shared unique id.
A real torch-free trainer would mint the id and drive rank 0 itself; here we
reuse the real send protocol by injecting the communicator into a sender
engine, exercising the worker-side UID path end to end.
"""
import pybase64 as base64
import torch
_set_ray_assigned_device()
from vllm.distributed.weight_transfer import ModuleSource
from vllm.distributed.weight_transfer.nccl_common import uid_init_process_group
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerWeightTransferEngine,
)
class NoopClient:
def init_weight_transfer_engine(self, init_info):
pass
def start_weight_update(self):
pass
def update_weights(self, update_info):
pass
def finish_weight_update(self, weight_version=None):
pass
dtype = getattr(torch, tensor_dtype)
model = torch.nn.Module()
test_module = torch.nn.Module()
test_module.register_parameter(
"weight",
torch.nn.Parameter(
torch.ones(tensor_shape, dtype=dtype, device="cuda"), requires_grad=False
),
)
model.add_module("test", test_module)
engine = NCCLTrainerWeightTransferEngine(
client=NoopClient(),
source=ModuleSource(model),
is_sender=True,
packed=False,
)
engine.model_update_group = uid_init_process_group(
base64.b64decode(uid_b64),
rank=0,
world_size=world_size,
device=torch.accelerator.current_device_index(),
)
try:
engine.send_weights()
torch.accelerator.synchronize()
return {"success": True}
finally:
engine.shutdown()
@ray.remote(num_gpus=1)
def trainer_broadcast_tensor_torch_free(
uid_b64: str,
world_size: int,
tensor_shape: list[int],
tensor_dtype: str,
) -> dict:
"""Trainer peer that uses no torch and no PyNcclCommunicator (a JAX stand-in).
Drives NCCL through the raw ctypes wrapper with a cupy send buffer, so it
exercises interop with a foreign NCCL peer rather than vLLM against itself.
``PyNcclCommunicator.__init__`` runs a one-element warm-up all_reduce right
after ``ncclCommInitRank``; that is a collective, so this peer must issue the
matching all_reduce before the broadcast or both ranks deadlock.
"""
import cupy as cp
import numpy as np
import pybase64 as base64
device = _set_ray_assigned_device()
cp.cuda.Device(device.index).use()
from vllm.distributed.device_communicators.pynccl_wrapper import (
NCCLLibrary,
buffer_type,
cudaStream_t,
ncclDataTypeEnum,
ncclRedOpTypeEnum,
)
assert tensor_dtype == "float32", "harness only wires float32"
# Fill on the host and copy up (a plain H2D memcpy) so we neither use torch
# nor make cupy JIT-compile a fill kernel (which needs NVRTC). All-ones lets
# the worker verify by sum == numel.
sendbuff = cp.asarray(np.ones(tensor_shape, dtype=np.float32))
warmup = cp.asarray(np.zeros(1, dtype=np.float32))
numel = int(sendbuff.size)
# The copies above run on the default stream; drain them before the NCCL
# collectives read the buffers on our own stream.
cp.cuda.Device(device.index).synchronize()
stream = cp.cuda.Stream()
nccl = NCCLLibrary()
comm = nccl.ncclCommInitRank(
world_size, nccl.unique_id_from_bytes(base64.b64decode(uid_b64)), 0
)
try:
# Mirror the worker's init-time warm-up all_reduce (rank 0 must join it).
nccl.ncclAllReduce(
buffer_type(warmup.data.ptr),
buffer_type(warmup.data.ptr),
1,
ncclDataTypeEnum.ncclFloat32,
ncclRedOpTypeEnum.ncclSum,
comm,
cudaStream_t(stream.ptr),
)
nccl.ncclBroadcast(
buffer_type(sendbuff.data.ptr),
buffer_type(sendbuff.data.ptr),
numel,
ncclDataTypeEnum.ncclFloat32,
0, # root
comm,
cudaStream_t(stream.ptr),
)
stream.synchronize()
return {"success": True}
finally:
# Abort (not the collective destroy) for an uncoordinated teardown.
nccl.ncclCommAbort(comm)
def _run_uid_transfer(trainer_task) -> None:
"""Mint a unique id, run the worker and the given trainer peer concurrently,
and assert the tensor round-trips. Both ranks must enter init together --
there is no store barrier on the UID path."""
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
_init_ray_for_weight_transfer()
nccl = NCCLLibrary()
uid_b64 = base64.b64encode(bytes(nccl.ncclGetUniqueId().internal)).decode()
world_size = 2 # 1 trainer + 1 inference worker
tensor_shape = [100, 100]
tensor_dtype = "float32"
inference_future = inference_receive_tensor.remote(
{
"rank_offset": 1,
"world_size": world_size,
"nccl_unique_id_b64": uid_b64,
"packed": False,
},
tensor_shape,
tensor_dtype,
)
trainer_future = trainer_task.remote(
uid_b64, world_size, tensor_shape, tensor_dtype
)
try:
trainer_result, result = ray.get(
[trainer_future, inference_future], timeout=120
)
finally:
ray.shutdown()
assert trainer_result["success"], "Trainer should complete successfully"
assert result["success"], (
f"Weight transfer failed. "
f"Received shape: {result['received_shape']}, "
f"Received sum: {result['received_sum']}"
)
@pytest.mark.skipif(
torch.accelerator.device_count() < 2,
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
)
def test_nccl_weight_transfer_between_processes_uid():
"""Weight transfer over a pre-shared ncclUniqueId (no TCPStore), with both
ranks on vLLM's ``PyNcclCommunicator``."""
_run_uid_transfer(trainer_broadcast_tensor_uid)
@pytest.mark.skipif(
torch.accelerator.device_count() < 2,
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
)
def test_nccl_weight_transfer_torch_free_trainer():
"""UID weight transfer against a foreign peer that uses no torch and no vLLM
communicator, only a cupy buffer and raw ``ncclBroadcast``."""
pytest.importorskip("cupy")
_run_uid_transfer(trainer_broadcast_tensor_torch_free)
@@ -11,6 +11,7 @@ from torch.distributed import ProcessGroup, ReduceOp
import vllm.envs as envs
from vllm.distributed.device_communicators.pynccl_wrapper import (
NCCL_UNIQUE_ID_BYTES,
NCCLLibrary,
buffer_type,
cudaStream_t,
@@ -60,6 +61,9 @@ def register_nccl_symmetric_ops(pynccl_comm):
class PyNcclCommunicator:
# None for communicators built via `from_unique_id_bytes` (no process group).
group: ProcessGroup | StatelessProcessGroup | None
def __init__(
self,
group: ProcessGroup | StatelessProcessGroup,
@@ -128,6 +132,14 @@ class PyNcclCommunicator:
self.unique_id.internal[i] = byte
else:
self.unique_id = group.broadcast_obj(self.unique_id, src=0)
self._init_comm(device)
def _init_comm(self, device: int | str | torch.device) -> None:
"""Create the communicator on `device` from the already-resolved
`self.unique_id` / `self.rank` / `self.world_size`, then run the
one-element warm-up all_reduce. Shared by `__init__` and
`from_unique_id_bytes` so the init handshake stays identical on both.
"""
if isinstance(device, int):
device = torch.device(f"cuda:{device}")
elif isinstance(device, str):
@@ -148,6 +160,62 @@ class PyNcclCommunicator:
stream.synchronize()
del data
@classmethod
def from_unique_id_bytes(
cls,
unique_id_bytes: bytes,
rank: int,
world_size: int,
device: int | str | torch.device,
library_path: str | None = None,
) -> "PyNcclCommunicator":
"""Build a communicator from pre-shared ``ncclUniqueId`` bytes.
For peers that cannot join a ``StatelessProcessGroup`` / TCPStore (e.g. a
torch-free JAX trainer): every rank passes the same id, minted once via
``ncclGetUniqueId`` and shared out of band. There is no barrier, so all
ranks must enter init concurrently or ``ncclCommInitRank`` hangs.
Warm-up handshake: immediately after ``ncclCommInitRank`` this issues a
one-element ``all_reduce`` (mirroring ``__init__``). It is a collective,
so every peer -- including a foreign, non-vLLM rank -- must issue a
matching one-element ``all_reduce`` before any other collective, or all
ranks deadlock.
"""
if len(unique_id_bytes) != NCCL_UNIQUE_ID_BYTES:
raise ValueError(
f"expected a {NCCL_UNIQUE_ID_BYTES}-byte NCCL unique id, "
f"got {len(unique_id_bytes)} bytes"
)
if not 0 <= rank < world_size:
raise ValueError(f"rank {rank} out of range for world_size {world_size}")
self = cls.__new__(cls)
self.rank = rank
self.world_size = world_size
self.group = None
if self.world_size == 1 or envs.VLLM_DISABLE_PYNCCL:
self.available = False
self.disabled = True
return self
try:
self.nccl = NCCLLibrary(library_path)
except Exception as e:
# Unlike the TCPStore path, silently disabling here leaves the peer
# blocked in ncclCommInitRank until timeout. The caller explicitly
# asked to join from a unique id, so fail loudly instead.
raise RuntimeError(
"failed to load the NCCL library for unique-id rendezvous"
) from e
self.available = True
self.disabled = False
self.nccl_version = self.nccl.ncclGetRawVersion()
self.unique_id = self.nccl.unique_id_from_bytes(unique_id_bytes)
self._init_comm(device)
return self
def destroy(self):
if self.available and not self.disabled:
# ncclCommAbort can block until all CUDA graphs that
@@ -51,6 +51,9 @@ class ncclUniqueId(ctypes.Structure):
_fields_ = [("internal", ctypes.c_byte * 128)]
NCCL_UNIQUE_ID_BYTES = ctypes.sizeof(ncclUniqueId)
# Mirror of NCCL's versioned ncclCommProperties_t (nccl_device/core.h,
# v2.31.2-1, 144 bytes). ncclCommQueryProperties fills fields gated by the
# version the caller declares in props.version, not by props.size, so never
+125 -8
View File
@@ -8,15 +8,19 @@ The dense (`NCCLWeightTransferEngine`) and sparse
sparse engine does not have to subclass the dense one.
"""
from dataclasses import dataclass
from dataclasses import asdict, dataclass, field
from typing import TYPE_CHECKING, Protocol
import pybase64 as base64
import torch
if TYPE_CHECKING:
from vllm.config.parallel import ParallelConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.device_communicators.pynccl_wrapper import (
NCCL_UNIQUE_ID_BYTES,
)
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
@@ -24,21 +28,102 @@ from vllm.distributed.weight_transfer.packed_tensor import (
)
@dataclass
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
"""Worker-side initialization info for NCCL-based weight transfer backends."""
def decode_nccl_unique_id(
*,
master_address: str | None,
master_port: int | None,
nccl_unique_id_b64: str | None,
ctx: str,
) -> bytes | None:
"""Validate the rendezvous mode and decode a pre-shared unique id."""
has_uid = bool(nccl_unique_id_b64)
has_addr = master_address is not None
has_port = master_port is not None
master_address: str
master_port: int
if has_uid and (has_addr or has_port):
raise ValueError(
f"{ctx}: pass nccl_unique_id_b64 OR master_address/master_port, not both"
)
if not has_uid:
if not (has_addr and has_port):
raise ValueError(
f"{ctx}: need nccl_unique_id_b64, or both master_address and "
f"master_port (got master_address={master_address!r}, "
f"master_port={master_port!r})"
)
return None
try:
decoded = base64.b64decode(nccl_unique_id_b64, validate=True)
except (ValueError, TypeError) as e:
raise ValueError(
f"{ctx}: nccl_unique_id_b64 is not valid standard base64 "
f"(URL-safe base64 is not accepted)"
) from e
if len(decoded) != NCCL_UNIQUE_ID_BYTES:
raise ValueError(
f"{ctx}: nccl_unique_id_b64 must decode to {NCCL_UNIQUE_ID_BYTES} "
f"bytes, got {len(decoded)}"
)
return decoded
@dataclass(kw_only=True)
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
"""Worker-side initialization info for NCCL-based weight transfer backends.
Keyword-only (`kw_only`): adding the optional `nccl_unique_id_b64` field
means the rendezvous fields can no longer keep a fixed positional slot, so a
stale positional call fails loudly instead of silently swapping arguments.
Provide exactly one rendezvous mode:
* ``master_address`` + ``master_port`` -- TCPStore / ``StatelessProcessGroup``
rendezvous (requires torch on every rank, including the trainer), or
* ``nccl_unique_id_b64`` -- standard (RFC 4648, *not* URL-safe) base64 of the
128 raw bytes from ``ncclGetUniqueId``, for torch-free trainers (e.g. JAX)
that mint the unique id out of band and share it (over HTTP, etc.). Note a
JAX peer must use ``base64.b64encode``, not ``urlsafe_b64encode``.
On the unique-id path all ranks must enter init concurrently (there is no
store barrier), and every peer must honor the warm-up handshake: the
worker's communicator issues a one-element ``all_reduce`` immediately after
``ncclCommInitRank`` (see ``PyNcclCommunicator.from_unique_id_bytes``), so a
foreign peer must issue a matching one-element ``all_reduce`` before any
other collective or all ranks deadlock.
"""
master_address: str | None = None
master_port: int | None = None
rank_offset: int
world_size: int
nccl_unique_id_b64: str | None = field(default=None, repr=False)
packed: bool = False
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
def __post_init__(self) -> None:
_ = self.nccl_unique_id_bytes
@property
def nccl_unique_id_bytes(self) -> bytes | None:
return decode_nccl_unique_id(
master_address=self.master_address,
master_port=self.master_port,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="NCCLWeightTransferInitInfo",
)
def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
only the mode actually in use. Shared by the dense and sparse trainer
engines so the two cannot drift."""
return {key: value for key, value in asdict(init_info).items() if value is not None}
class NCCLRendezvous(Protocol):
"""The three fields `trainer_init` needs to open the trainer endpoint.
"""The TCP rendezvous fields `trainer_init` needs.
Structural so each backend can keep its own trainer init info next to its
engine (dense `NCCLTrainerInitInfo`, `SparseNCCLTrainerInitInfo`) without
@@ -73,6 +158,27 @@ def stateless_init_process_group(
return PyNcclCommunicator(pg, device=device)
def uid_init_process_group(
nccl_unique_id_bytes: bytes,
rank: int,
world_size: int,
device,
) -> "PyNcclCommunicator":
"""Join the NCCL group from pre-shared ``ncclUniqueId`` bytes.
The torch-free rendezvous alternative to `stateless_init_process_group`: no
TCPStore, and therefore no barrier -- every rank must enter concurrently.
"""
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
return PyNcclCommunicator.from_unique_id_bytes(
nccl_unique_id_bytes,
rank=rank,
world_size=world_size,
device=device,
)
def worker_init_process_group(
init_info: NCCLWeightTransferInitInfo,
parallel_config: "ParallelConfig",
@@ -80,7 +186,7 @@ def worker_init_process_group(
"""Create the trainer<->worker NCCL group on an inference worker.
Computes a unique rank for this worker across all data-parallel groups and
joins the stateless process group with the trainer.
joins the trainer via whichever rendezvous mode `init_info` carries.
"""
# Calculate the global rank in the trainer-worker process group.
# Must account for data parallel to get unique ranks across all workers.
@@ -93,6 +199,17 @@ def worker_init_process_group(
rank = worker_rank + init_info.rank_offset
device = torch.accelerator.current_device_index()
unique_id_bytes = init_info.nccl_unique_id_bytes
if unique_id_bytes is not None:
return uid_init_process_group(
unique_id_bytes,
rank,
init_info.world_size,
device,
)
# __post_init__ guarantees the TCP pair when there is no unique id.
assert init_info.master_address is not None
assert init_info.master_port is not None
return stateless_init_process_group(
init_info.master_address,
init_info.master_port,
@@ -26,6 +26,7 @@ from vllm.distributed.weight_transfer.base import (
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
worker_init_payload,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_common import (
@@ -57,6 +58,13 @@ class NCCLTrainerInitInfo(TrainerInitInfo):
`world_size` is the full trainer+worker NCCL group size. `rank` (from
`TrainerInitInfo`) identifies this trainer process; rank 0 is the sender.
The trainer joins over a TCPStore rendezvous (`master_address` +
`master_port`). Torch-free trainers (e.g. JAX) that cannot join a TCPStore
mint an `ncclUniqueId` themselves and drive rank 0 out of band; they ship a
`nccl_unique_id_b64` payload straight to the inference workers'
`init_weight_transfer_engine` (see `NCCLWeightTransferInitInfo`) rather than
going through this engine.
`packed` / buffer sizes are the transfer's wire params. The trainer
propagates them to the worker at `trainer_init` so the two sides cannot
disagree. Note this defaults to packed, unlike the worker-side
@@ -301,7 +309,8 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni
# open the trainer endpoint (rank 0); both sides must rendezvous together.
with ThreadPoolExecutor(max_workers=1) as exe:
future = exe.submit(
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
engine.client.init_weight_transfer_engine,
worker_init_payload(worker_init_info),
)
engine.model_update_group = open_trainer_endpoint(init_info)
future.result() # surface any inference-side init error
@@ -31,6 +31,7 @@ from vllm.distributed.weight_transfer.base import (
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
worker_init_payload,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_common import (
@@ -256,7 +257,8 @@ class SparseNCCLTrainerWeightTransferEngine(
# open the trainer endpoint (rank 0); both sides must rendezvous together.
with ThreadPoolExecutor(max_workers=1) as exe:
future = exe.submit(
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
engine.client.init_weight_transfer_engine,
worker_init_payload(worker_init_info),
)
# Open the trainer endpoint as NCCL rank 0 on the current device
# (the init info satisfies the helper's rendezvous protocol).