From d0e695a91b67a8214c2e9ed77595186d2f2844b6 Mon Sep 17 00:00:00 2001 From: dharak-cohere Date: Tue, 1 Sep 2026 02:55:57 -0400 Subject: [PATCH] [Distributed] Support pre-shared ncclUniqueId rendezvous for weight transfer (#53784) Signed-off-by: Dharak Kharod Co-authored-by: Cursor --- .../test_weight_transfer_nccl_uid.py | 465 ++++++++++++++++++ .../device_communicators/pynccl.py | 68 +++ .../device_communicators/pynccl_wrapper.py | 3 + .../weight_transfer/nccl_common.py | 133 ++++- .../weight_transfer/nccl_engine.py | 11 +- .../weight_transfer/sparse_nccl_engine.py | 4 +- 6 files changed, 674 insertions(+), 10 deletions(-) create mode 100644 tests/distributed/test_weight_transfer_nccl_uid.py diff --git a/tests/distributed/test_weight_transfer_nccl_uid.py b/tests/distributed/test_weight_transfer_nccl_uid.py new file mode 100644 index 0000000000..18a6442a2a --- /dev/null +++ b/tests/distributed/test_weight_transfer_nccl_uid.py @@ -0,0 +1,465 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the pre-shared ncclUniqueId weight-transfer rendezvous. + +The rendezvous lets a peer that cannot join a TCPStore (e.g. a torch-free JAX +trainer) initialize NCCL weight transfer from a unique id minted out of band. + +* Unit tests for the mutually-exclusive rendezvous-mode validation on the NCCL + init infos. +* Two-GPU integration tests driving the real worker engine against a peer: one + via vLLM's own ``PyNcclCommunicator``, one via a genuinely torch-free trainer + (cupy buffer + raw ``ncclBroadcast``). + +The shared Ray helpers come from the sibling module (unchanged); the worker task +is local so the sibling's TCP test and its worker stay untouched. +""" + +from unittest.mock import MagicMock + +import pybase64 as base64 +import pytest +import ray +import torch + +from tests.distributed.test_weight_transfer import ( + _init_ray_for_weight_transfer, + _set_ray_assigned_device, + create_mock_vllm_config, +) +from vllm.config.weight_transfer import WeightTransferConfig +from vllm.distributed.weight_transfer.nccl_engine import ( + NCCLTrainerInitInfo, + NCCLWeightTransferEngine, + NCCLWeightTransferInitInfo, +) + +# Standard base64 of exactly 128 zero bytes: a well-formed (dummy) ncclUniqueId. +VALID_UID_B64 = base64.b64encode(b"\x00" * 128).decode() + + +class TestNCCLRendezvousValidation: + """Test the xor-of-rendezvous-modes validation on the NCCL init infos.""" + + def test_worker_init_info_is_keyword_only(self): + # kw_only so a stale positional call fails loudly rather than silently + # swapping fields once the rendezvous fields became optional. + with pytest.raises(TypeError): + NCCLWeightTransferInitInfo(1, 3, "127.0.0.1", 12345) # type: ignore[call-arg] + + def test_trainer_init_info_accepts_positional_fields(self): + # NCCLTrainerInitInfo keeps positional subclass fields; only `rank` + # (from TrainerInitInfo) is keyword-only. + trainer_positional = NCCLTrainerInitInfo("127.0.0.1", 12345, 3, rank=0) + trainer_keyword = NCCLTrainerInitInfo( + master_address="127.0.0.1", + master_port=12345, + world_size=3, + rank=0, + ) + assert trainer_positional == trainer_keyword + + def test_tcp_mode_valid(self): + info = NCCLWeightTransferInitInfo( + rank_offset=1, + world_size=3, + master_address="127.0.0.1", + master_port=12345, + ) + assert info.master_address == "127.0.0.1" + assert info.nccl_unique_id_b64 is None + + def test_uid_mode_valid(self): + info = NCCLWeightTransferInitInfo( + rank_offset=1, + world_size=3, + nccl_unique_id_b64=VALID_UID_B64, + ) + assert info.nccl_unique_id_b64 == VALID_UID_B64 + assert info.master_address is None + assert info.master_port is None + assert VALID_UID_B64 not in repr(info) + + def test_both_modes_raises(self): + with pytest.raises(ValueError, match="not both"): + NCCLWeightTransferInitInfo( + rank_offset=1, + world_size=3, + master_address="127.0.0.1", + master_port=12345, + nccl_unique_id_b64=VALID_UID_B64, + ) + + def test_neither_mode_raises(self): + with pytest.raises(ValueError, match="need nccl_unique_id_b64"): + NCCLWeightTransferInitInfo(rank_offset=1, world_size=3) + + def test_half_specified_tcp_raises(self): + with pytest.raises(ValueError, match="master_port"): + NCCLWeightTransferInitInfo( + rank_offset=1, + world_size=3, + master_address="127.0.0.1", + ) + + def test_bad_base64_uid_raises(self): + with pytest.raises(ValueError, match="not valid standard base64"): + NCCLWeightTransferInitInfo( + rank_offset=1, + world_size=3, + nccl_unique_id_b64="!!!!", + ) + + def test_wrong_length_uid_raises(self): + short_uid = base64.b64encode(b"\x00" * 64).decode() + with pytest.raises(ValueError, match="128"): + NCCLWeightTransferInitInfo( + rank_offset=1, + world_size=3, + nccl_unique_id_b64=short_uid, + ) + + def test_parse_init_info_uid_valid(self): + engine = NCCLWeightTransferEngine( + WeightTransferConfig(backend="nccl"), + create_mock_vllm_config(), + torch.device("cuda"), + MagicMock(spec=torch.nn.Module), + ) + init_info = engine.parse_init_info( + { + "nccl_unique_id_b64": VALID_UID_B64, + "rank_offset": 1, + "world_size": 3, + } + ) + assert isinstance(init_info, NCCLWeightTransferInitInfo) + assert init_info.nccl_unique_id_b64 == VALID_UID_B64 + assert init_info.master_address is None + + def test_parse_init_info_both_modes_raises(self): + engine = NCCLWeightTransferEngine( + WeightTransferConfig(backend="nccl"), + create_mock_vllm_config(), + torch.device("cuda"), + MagicMock(spec=torch.nn.Module), + ) + with pytest.raises(ValueError, match="not both"): + engine.parse_init_info( + { + "master_address": "127.0.0.1", + "master_port": 12345, + "nccl_unique_id_b64": VALID_UID_B64, + "rank_offset": 1, + "world_size": 3, + } + ) + + @pytest.mark.parametrize( + ("unique_id_bytes", "rank", "world_size", "match"), + [ + (b"\x00" * 64, 0, 2, "128-byte"), + (b"\x00" * 128, -1, 2, "out of range"), + (b"\x00" * 128, 2, 2, "out of range"), + ], + ) + def test_from_unique_id_bytes_validation( + self, unique_id_bytes, rank, world_size, match + ): + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator + + with pytest.raises(ValueError, match=match): + PyNcclCommunicator.from_unique_id_bytes( + unique_id_bytes, + rank=rank, + world_size=world_size, + device=0, + ) + + +@ray.remote(num_gpus=1) +def inference_receive_tensor( + init_info_dict: dict, + tensor_shape: list[int], + tensor_dtype: str, +) -> dict: + """Worker that joins via the given init dict and records the tensor it gets. + + Local to this module (rather than the sibling's TCP-only worker) so it can + take a UID init payload -- the same dict boundary an external trainer POSTs. + """ + import contextlib + + import torch + + _set_ray_assigned_device() + + from vllm.config.parallel import ParallelConfig + from vllm.config.weight_transfer import WeightTransferConfig + from vllm.distributed.weight_transfer.nccl_engine import ( + NCCLWeightTransferEngine, + NCCLWeightTransferInitInfo, + NCCLWeightTransferUpdateInfo, + ) + + class Recorder(torch.nn.Module): + def __init__(self): + super().__init__() + self.received = [] + + def load_weights(self, weights): + for name, tensor in weights: + self.received.append((name, tensor.clone())) + + config = WeightTransferConfig(backend="nccl") + vllm_config = MagicMock() + parallel_config = MagicMock(spec=ParallelConfig) + parallel_config.rank = 0 + parallel_config.world_size = 1 + parallel_config.data_parallel_rank = 0 + parallel_config.data_parallel_index = 0 + vllm_config.parallel_config = parallel_config + vllm_config.model_config = MagicMock() + + recorder = Recorder() + engine = NCCLWeightTransferEngine( + config, vllm_config, torch.device("cuda"), recorder + ) + # Transport-only test: bypass the set_current_vllm_config context that + # receive_weights enters, since vllm_config here is a mock. + import vllm.config as _vllm_config_mod + + _vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext() + + try: + engine.init_transfer_engine(NCCLWeightTransferInitInfo(**init_info_dict)) + engine.receive_weights( + NCCLWeightTransferUpdateInfo( + names=["test.weight"], + dtype_names=[tensor_dtype], + shapes=[tensor_shape], + ) + ) + torch.accelerator.synchronize() + + received_shape = None + received_sum = None + if len(recorder.received) == 1: + _name, tensor = recorder.received[0] + received_shape = list(tensor.shape) + received_sum = tensor.sum().item() + + return { + "success": received_shape == tensor_shape + and abs(received_sum - torch.tensor(tensor_shape).prod().item()) < 0.01, + "received_shape": received_shape, + "received_sum": received_sum, + } + finally: + engine.shutdown() + + +@ray.remote(num_gpus=1) +def trainer_broadcast_tensor_uid( + uid_b64: str, + world_size: int, + tensor_shape: list[int], + tensor_dtype: str, +) -> dict: + """Drive rank 0 via vLLM's own communicator from a pre-shared unique id. + + A real torch-free trainer would mint the id and drive rank 0 itself; here we + reuse the real send protocol by injecting the communicator into a sender + engine, exercising the worker-side UID path end to end. + """ + import pybase64 as base64 + import torch + + _set_ray_assigned_device() + + from vllm.distributed.weight_transfer import ModuleSource + from vllm.distributed.weight_transfer.nccl_common import uid_init_process_group + from vllm.distributed.weight_transfer.nccl_engine import ( + NCCLTrainerWeightTransferEngine, + ) + + class NoopClient: + def init_weight_transfer_engine(self, init_info): + pass + + def start_weight_update(self): + pass + + def update_weights(self, update_info): + pass + + def finish_weight_update(self, weight_version=None): + pass + + dtype = getattr(torch, tensor_dtype) + model = torch.nn.Module() + test_module = torch.nn.Module() + test_module.register_parameter( + "weight", + torch.nn.Parameter( + torch.ones(tensor_shape, dtype=dtype, device="cuda"), requires_grad=False + ), + ) + model.add_module("test", test_module) + + engine = NCCLTrainerWeightTransferEngine( + client=NoopClient(), + source=ModuleSource(model), + is_sender=True, + packed=False, + ) + engine.model_update_group = uid_init_process_group( + base64.b64decode(uid_b64), + rank=0, + world_size=world_size, + device=torch.accelerator.current_device_index(), + ) + try: + engine.send_weights() + torch.accelerator.synchronize() + return {"success": True} + finally: + engine.shutdown() + + +@ray.remote(num_gpus=1) +def trainer_broadcast_tensor_torch_free( + uid_b64: str, + world_size: int, + tensor_shape: list[int], + tensor_dtype: str, +) -> dict: + """Trainer peer that uses no torch and no PyNcclCommunicator (a JAX stand-in). + + Drives NCCL through the raw ctypes wrapper with a cupy send buffer, so it + exercises interop with a foreign NCCL peer rather than vLLM against itself. + ``PyNcclCommunicator.__init__`` runs a one-element warm-up all_reduce right + after ``ncclCommInitRank``; that is a collective, so this peer must issue the + matching all_reduce before the broadcast or both ranks deadlock. + """ + import cupy as cp + import numpy as np + import pybase64 as base64 + + device = _set_ray_assigned_device() + cp.cuda.Device(device.index).use() + + from vllm.distributed.device_communicators.pynccl_wrapper import ( + NCCLLibrary, + buffer_type, + cudaStream_t, + ncclDataTypeEnum, + ncclRedOpTypeEnum, + ) + + assert tensor_dtype == "float32", "harness only wires float32" + # Fill on the host and copy up (a plain H2D memcpy) so we neither use torch + # nor make cupy JIT-compile a fill kernel (which needs NVRTC). All-ones lets + # the worker verify by sum == numel. + sendbuff = cp.asarray(np.ones(tensor_shape, dtype=np.float32)) + warmup = cp.asarray(np.zeros(1, dtype=np.float32)) + numel = int(sendbuff.size) + # The copies above run on the default stream; drain them before the NCCL + # collectives read the buffers on our own stream. + cp.cuda.Device(device.index).synchronize() + stream = cp.cuda.Stream() + + nccl = NCCLLibrary() + comm = nccl.ncclCommInitRank( + world_size, nccl.unique_id_from_bytes(base64.b64decode(uid_b64)), 0 + ) + try: + # Mirror the worker's init-time warm-up all_reduce (rank 0 must join it). + nccl.ncclAllReduce( + buffer_type(warmup.data.ptr), + buffer_type(warmup.data.ptr), + 1, + ncclDataTypeEnum.ncclFloat32, + ncclRedOpTypeEnum.ncclSum, + comm, + cudaStream_t(stream.ptr), + ) + nccl.ncclBroadcast( + buffer_type(sendbuff.data.ptr), + buffer_type(sendbuff.data.ptr), + numel, + ncclDataTypeEnum.ncclFloat32, + 0, # root + comm, + cudaStream_t(stream.ptr), + ) + stream.synchronize() + return {"success": True} + finally: + # Abort (not the collective destroy) for an uncoordinated teardown. + nccl.ncclCommAbort(comm) + + +def _run_uid_transfer(trainer_task) -> None: + """Mint a unique id, run the worker and the given trainer peer concurrently, + and assert the tensor round-trips. Both ranks must enter init together -- + there is no store barrier on the UID path.""" + from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary + + _init_ray_for_weight_transfer() + + nccl = NCCLLibrary() + uid_b64 = base64.b64encode(bytes(nccl.ncclGetUniqueId().internal)).decode() + + world_size = 2 # 1 trainer + 1 inference worker + tensor_shape = [100, 100] + tensor_dtype = "float32" + + inference_future = inference_receive_tensor.remote( + { + "rank_offset": 1, + "world_size": world_size, + "nccl_unique_id_b64": uid_b64, + "packed": False, + }, + tensor_shape, + tensor_dtype, + ) + trainer_future = trainer_task.remote( + uid_b64, world_size, tensor_shape, tensor_dtype + ) + + try: + trainer_result, result = ray.get( + [trainer_future, inference_future], timeout=120 + ) + finally: + ray.shutdown() + + assert trainer_result["success"], "Trainer should complete successfully" + assert result["success"], ( + f"Weight transfer failed. " + f"Received shape: {result['received_shape']}, " + f"Received sum: {result['received_sum']}" + ) + + +@pytest.mark.skipif( + torch.accelerator.device_count() < 2, + reason="Need at least 2 GPUs to run NCCL weight transfer test.", +) +def test_nccl_weight_transfer_between_processes_uid(): + """Weight transfer over a pre-shared ncclUniqueId (no TCPStore), with both + ranks on vLLM's ``PyNcclCommunicator``.""" + _run_uid_transfer(trainer_broadcast_tensor_uid) + + +@pytest.mark.skipif( + torch.accelerator.device_count() < 2, + reason="Need at least 2 GPUs to run NCCL weight transfer test.", +) +def test_nccl_weight_transfer_torch_free_trainer(): + """UID weight transfer against a foreign peer that uses no torch and no vLLM + communicator, only a cupy buffer and raw ``ncclBroadcast``.""" + pytest.importorskip("cupy") + _run_uid_transfer(trainer_broadcast_tensor_torch_free) diff --git a/vllm/distributed/device_communicators/pynccl.py b/vllm/distributed/device_communicators/pynccl.py index 9c8a3f6005..916d0308bf 100644 --- a/vllm/distributed/device_communicators/pynccl.py +++ b/vllm/distributed/device_communicators/pynccl.py @@ -11,6 +11,7 @@ from torch.distributed import ProcessGroup, ReduceOp import vllm.envs as envs from vllm.distributed.device_communicators.pynccl_wrapper import ( + NCCL_UNIQUE_ID_BYTES, NCCLLibrary, buffer_type, cudaStream_t, @@ -60,6 +61,9 @@ def register_nccl_symmetric_ops(pynccl_comm): class PyNcclCommunicator: + # None for communicators built via `from_unique_id_bytes` (no process group). + group: ProcessGroup | StatelessProcessGroup | None + def __init__( self, group: ProcessGroup | StatelessProcessGroup, @@ -128,6 +132,14 @@ class PyNcclCommunicator: self.unique_id.internal[i] = byte else: self.unique_id = group.broadcast_obj(self.unique_id, src=0) + self._init_comm(device) + + def _init_comm(self, device: int | str | torch.device) -> None: + """Create the communicator on `device` from the already-resolved + `self.unique_id` / `self.rank` / `self.world_size`, then run the + one-element warm-up all_reduce. Shared by `__init__` and + `from_unique_id_bytes` so the init handshake stays identical on both. + """ if isinstance(device, int): device = torch.device(f"cuda:{device}") elif isinstance(device, str): @@ -148,6 +160,62 @@ class PyNcclCommunicator: stream.synchronize() del data + @classmethod + def from_unique_id_bytes( + cls, + unique_id_bytes: bytes, + rank: int, + world_size: int, + device: int | str | torch.device, + library_path: str | None = None, + ) -> "PyNcclCommunicator": + """Build a communicator from pre-shared ``ncclUniqueId`` bytes. + + For peers that cannot join a ``StatelessProcessGroup`` / TCPStore (e.g. a + torch-free JAX trainer): every rank passes the same id, minted once via + ``ncclGetUniqueId`` and shared out of band. There is no barrier, so all + ranks must enter init concurrently or ``ncclCommInitRank`` hangs. + + Warm-up handshake: immediately after ``ncclCommInitRank`` this issues a + one-element ``all_reduce`` (mirroring ``__init__``). It is a collective, + so every peer -- including a foreign, non-vLLM rank -- must issue a + matching one-element ``all_reduce`` before any other collective, or all + ranks deadlock. + """ + if len(unique_id_bytes) != NCCL_UNIQUE_ID_BYTES: + raise ValueError( + f"expected a {NCCL_UNIQUE_ID_BYTES}-byte NCCL unique id, " + f"got {len(unique_id_bytes)} bytes" + ) + if not 0 <= rank < world_size: + raise ValueError(f"rank {rank} out of range for world_size {world_size}") + + self = cls.__new__(cls) + self.rank = rank + self.world_size = world_size + self.group = None + + if self.world_size == 1 or envs.VLLM_DISABLE_PYNCCL: + self.available = False + self.disabled = True + return self + try: + self.nccl = NCCLLibrary(library_path) + except Exception as e: + # Unlike the TCPStore path, silently disabling here leaves the peer + # blocked in ncclCommInitRank until timeout. The caller explicitly + # asked to join from a unique id, so fail loudly instead. + raise RuntimeError( + "failed to load the NCCL library for unique-id rendezvous" + ) from e + + self.available = True + self.disabled = False + self.nccl_version = self.nccl.ncclGetRawVersion() + self.unique_id = self.nccl.unique_id_from_bytes(unique_id_bytes) + self._init_comm(device) + return self + def destroy(self): if self.available and not self.disabled: # ncclCommAbort can block until all CUDA graphs that diff --git a/vllm/distributed/device_communicators/pynccl_wrapper.py b/vllm/distributed/device_communicators/pynccl_wrapper.py index 03ac549dd8..4d43371da1 100644 --- a/vllm/distributed/device_communicators/pynccl_wrapper.py +++ b/vllm/distributed/device_communicators/pynccl_wrapper.py @@ -51,6 +51,9 @@ class ncclUniqueId(ctypes.Structure): _fields_ = [("internal", ctypes.c_byte * 128)] +NCCL_UNIQUE_ID_BYTES = ctypes.sizeof(ncclUniqueId) + + # Mirror of NCCL's versioned ncclCommProperties_t (nccl_device/core.h, # v2.31.2-1, 144 bytes). ncclCommQueryProperties fills fields gated by the # version the caller declares in props.version, not by props.size, so never diff --git a/vllm/distributed/weight_transfer/nccl_common.py b/vllm/distributed/weight_transfer/nccl_common.py index de72063c32..67e578640a 100644 --- a/vllm/distributed/weight_transfer/nccl_common.py +++ b/vllm/distributed/weight_transfer/nccl_common.py @@ -8,15 +8,19 @@ The dense (`NCCLWeightTransferEngine`) and sparse sparse engine does not have to subclass the dense one. """ -from dataclasses import dataclass +from dataclasses import asdict, dataclass, field from typing import TYPE_CHECKING, Protocol +import pybase64 as base64 import torch if TYPE_CHECKING: from vllm.config.parallel import ParallelConfig from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator +from vllm.distributed.device_communicators.pynccl_wrapper import ( + NCCL_UNIQUE_ID_BYTES, +) from vllm.distributed.weight_transfer.base import WeightTransferInitInfo from vllm.distributed.weight_transfer.packed_tensor import ( DEFAULT_PACKED_BUFFER_SIZE_BYTES, @@ -24,21 +28,102 @@ from vllm.distributed.weight_transfer.packed_tensor import ( ) -@dataclass -class NCCLWeightTransferInitInfo(WeightTransferInitInfo): - """Worker-side initialization info for NCCL-based weight transfer backends.""" +def decode_nccl_unique_id( + *, + master_address: str | None, + master_port: int | None, + nccl_unique_id_b64: str | None, + ctx: str, +) -> bytes | None: + """Validate the rendezvous mode and decode a pre-shared unique id.""" + has_uid = bool(nccl_unique_id_b64) + has_addr = master_address is not None + has_port = master_port is not None - master_address: str - master_port: int + if has_uid and (has_addr or has_port): + raise ValueError( + f"{ctx}: pass nccl_unique_id_b64 OR master_address/master_port, not both" + ) + if not has_uid: + if not (has_addr and has_port): + raise ValueError( + f"{ctx}: need nccl_unique_id_b64, or both master_address and " + f"master_port (got master_address={master_address!r}, " + f"master_port={master_port!r})" + ) + return None + try: + decoded = base64.b64decode(nccl_unique_id_b64, validate=True) + except (ValueError, TypeError) as e: + raise ValueError( + f"{ctx}: nccl_unique_id_b64 is not valid standard base64 " + f"(URL-safe base64 is not accepted)" + ) from e + if len(decoded) != NCCL_UNIQUE_ID_BYTES: + raise ValueError( + f"{ctx}: nccl_unique_id_b64 must decode to {NCCL_UNIQUE_ID_BYTES} " + f"bytes, got {len(decoded)}" + ) + return decoded + + +@dataclass(kw_only=True) +class NCCLWeightTransferInitInfo(WeightTransferInitInfo): + """Worker-side initialization info for NCCL-based weight transfer backends. + + Keyword-only (`kw_only`): adding the optional `nccl_unique_id_b64` field + means the rendezvous fields can no longer keep a fixed positional slot, so a + stale positional call fails loudly instead of silently swapping arguments. + + Provide exactly one rendezvous mode: + + * ``master_address`` + ``master_port`` -- TCPStore / ``StatelessProcessGroup`` + rendezvous (requires torch on every rank, including the trainer), or + * ``nccl_unique_id_b64`` -- standard (RFC 4648, *not* URL-safe) base64 of the + 128 raw bytes from ``ncclGetUniqueId``, for torch-free trainers (e.g. JAX) + that mint the unique id out of band and share it (over HTTP, etc.). Note a + JAX peer must use ``base64.b64encode``, not ``urlsafe_b64encode``. + + On the unique-id path all ranks must enter init concurrently (there is no + store barrier), and every peer must honor the warm-up handshake: the + worker's communicator issues a one-element ``all_reduce`` immediately after + ``ncclCommInitRank`` (see ``PyNcclCommunicator.from_unique_id_bytes``), so a + foreign peer must issue a matching one-element ``all_reduce`` before any + other collective or all ranks deadlock. + """ + + master_address: str | None = None + master_port: int | None = None rank_offset: int world_size: int + nccl_unique_id_b64: str | None = field(default=None, repr=False) packed: bool = False packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS + def __post_init__(self) -> None: + _ = self.nccl_unique_id_bytes + + @property + def nccl_unique_id_bytes(self) -> bytes | None: + return decode_nccl_unique_id( + master_address=self.master_address, + master_port=self.master_port, + nccl_unique_id_b64=self.nccl_unique_id_b64, + ctx="NCCLWeightTransferInitInfo", + ) + + +def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict: + """Serialize a worker init info for `init_weight_transfer_engine`, dropping + the unset rendezvous field (the UID in TCP mode) so the wire payload carries + only the mode actually in use. Shared by the dense and sparse trainer + engines so the two cannot drift.""" + return {key: value for key, value in asdict(init_info).items() if value is not None} + class NCCLRendezvous(Protocol): - """The three fields `trainer_init` needs to open the trainer endpoint. + """The TCP rendezvous fields `trainer_init` needs. Structural so each backend can keep its own trainer init info next to its engine (dense `NCCLTrainerInitInfo`, `SparseNCCLTrainerInitInfo`) without @@ -73,6 +158,27 @@ def stateless_init_process_group( return PyNcclCommunicator(pg, device=device) +def uid_init_process_group( + nccl_unique_id_bytes: bytes, + rank: int, + world_size: int, + device, +) -> "PyNcclCommunicator": + """Join the NCCL group from pre-shared ``ncclUniqueId`` bytes. + + The torch-free rendezvous alternative to `stateless_init_process_group`: no + TCPStore, and therefore no barrier -- every rank must enter concurrently. + """ + from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator + + return PyNcclCommunicator.from_unique_id_bytes( + nccl_unique_id_bytes, + rank=rank, + world_size=world_size, + device=device, + ) + + def worker_init_process_group( init_info: NCCLWeightTransferInitInfo, parallel_config: "ParallelConfig", @@ -80,7 +186,7 @@ def worker_init_process_group( """Create the trainer<->worker NCCL group on an inference worker. Computes a unique rank for this worker across all data-parallel groups and - joins the stateless process group with the trainer. + joins the trainer via whichever rendezvous mode `init_info` carries. """ # Calculate the global rank in the trainer-worker process group. # Must account for data parallel to get unique ranks across all workers. @@ -93,6 +199,17 @@ def worker_init_process_group( rank = worker_rank + init_info.rank_offset device = torch.accelerator.current_device_index() + unique_id_bytes = init_info.nccl_unique_id_bytes + if unique_id_bytes is not None: + return uid_init_process_group( + unique_id_bytes, + rank, + init_info.world_size, + device, + ) + # __post_init__ guarantees the TCP pair when there is no unique id. + assert init_info.master_address is not None + assert init_info.master_port is not None return stateless_init_process_group( init_info.master_address, init_info.master_port, diff --git a/vllm/distributed/weight_transfer/nccl_engine.py b/vllm/distributed/weight_transfer/nccl_engine.py index d0b8c22c5b..1efd8180df 100644 --- a/vllm/distributed/weight_transfer/nccl_engine.py +++ b/vllm/distributed/weight_transfer/nccl_engine.py @@ -26,6 +26,7 @@ from vllm.distributed.weight_transfer.base import ( ) from vllm.distributed.weight_transfer.nccl_common import ( NCCLWeightTransferInitInfo, + worker_init_payload, worker_init_process_group, ) from vllm.distributed.weight_transfer.nccl_common import ( @@ -57,6 +58,13 @@ class NCCLTrainerInitInfo(TrainerInitInfo): `world_size` is the full trainer+worker NCCL group size. `rank` (from `TrainerInitInfo`) identifies this trainer process; rank 0 is the sender. + The trainer joins over a TCPStore rendezvous (`master_address` + + `master_port`). Torch-free trainers (e.g. JAX) that cannot join a TCPStore + mint an `ncclUniqueId` themselves and drive rank 0 out of band; they ship a + `nccl_unique_id_b64` payload straight to the inference workers' + `init_weight_transfer_engine` (see `NCCLWeightTransferInitInfo`) rather than + going through this engine. + `packed` / buffer sizes are the transfer's wire params. The trainer propagates them to the worker at `trainer_init` so the two sides cannot disagree. Note this defaults to packed, unlike the worker-side @@ -301,7 +309,8 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni # open the trainer endpoint (rank 0); both sides must rendezvous together. with ThreadPoolExecutor(max_workers=1) as exe: future = exe.submit( - engine.client.init_weight_transfer_engine, asdict(worker_init_info) + engine.client.init_weight_transfer_engine, + worker_init_payload(worker_init_info), ) engine.model_update_group = open_trainer_endpoint(init_info) future.result() # surface any inference-side init error diff --git a/vllm/distributed/weight_transfer/sparse_nccl_engine.py b/vllm/distributed/weight_transfer/sparse_nccl_engine.py index 89eb892a67..9f91da3418 100644 --- a/vllm/distributed/weight_transfer/sparse_nccl_engine.py +++ b/vllm/distributed/weight_transfer/sparse_nccl_engine.py @@ -31,6 +31,7 @@ from vllm.distributed.weight_transfer.base import ( ) from vllm.distributed.weight_transfer.nccl_common import ( NCCLWeightTransferInitInfo, + worker_init_payload, worker_init_process_group, ) from vllm.distributed.weight_transfer.nccl_common import ( @@ -256,7 +257,8 @@ class SparseNCCLTrainerWeightTransferEngine( # open the trainer endpoint (rank 0); both sides must rendezvous together. with ThreadPoolExecutor(max_workers=1) as exe: future = exe.submit( - engine.client.init_weight_transfer_engine, asdict(worker_init_info) + engine.client.init_weight_transfer_engine, + worker_init_payload(worker_init_info), ) # Open the trainer endpoint as NCCL rank 0 on the current device # (the init info satisfies the helper's rendezvous protocol).