[Distributed] Support pre-shared ncclUniqueId rendezvous for weight transfer (#53784)

Signed-off-by: Dharak Kharod <dharak.kharod@cohere.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
dharak-cohere
2026-09-01 14:55:57 +08:00
committed by GitHub
co-authored by Cursor
parent fa99a6fea6
commit d0e695a91b
6 changed files with 674 additions and 10 deletions
@@ -11,6 +11,7 @@ from torch.distributed import ProcessGroup, ReduceOp
import vllm.envs as envs
from vllm.distributed.device_communicators.pynccl_wrapper import (
NCCL_UNIQUE_ID_BYTES,
NCCLLibrary,
buffer_type,
cudaStream_t,
@@ -60,6 +61,9 @@ def register_nccl_symmetric_ops(pynccl_comm):
class PyNcclCommunicator:
# None for communicators built via `from_unique_id_bytes` (no process group).
group: ProcessGroup | StatelessProcessGroup | None
def __init__(
self,
group: ProcessGroup | StatelessProcessGroup,
@@ -128,6 +132,14 @@ class PyNcclCommunicator:
self.unique_id.internal[i] = byte
else:
self.unique_id = group.broadcast_obj(self.unique_id, src=0)
self._init_comm(device)
def _init_comm(self, device: int | str | torch.device) -> None:
"""Create the communicator on `device` from the already-resolved
`self.unique_id` / `self.rank` / `self.world_size`, then run the
one-element warm-up all_reduce. Shared by `__init__` and
`from_unique_id_bytes` so the init handshake stays identical on both.
"""
if isinstance(device, int):
device = torch.device(f"cuda:{device}")
elif isinstance(device, str):
@@ -148,6 +160,62 @@ class PyNcclCommunicator:
stream.synchronize()
del data
@classmethod
def from_unique_id_bytes(
cls,
unique_id_bytes: bytes,
rank: int,
world_size: int,
device: int | str | torch.device,
library_path: str | None = None,
) -> "PyNcclCommunicator":
"""Build a communicator from pre-shared ``ncclUniqueId`` bytes.
For peers that cannot join a ``StatelessProcessGroup`` / TCPStore (e.g. a
torch-free JAX trainer): every rank passes the same id, minted once via
``ncclGetUniqueId`` and shared out of band. There is no barrier, so all
ranks must enter init concurrently or ``ncclCommInitRank`` hangs.
Warm-up handshake: immediately after ``ncclCommInitRank`` this issues a
one-element ``all_reduce`` (mirroring ``__init__``). It is a collective,
so every peer -- including a foreign, non-vLLM rank -- must issue a
matching one-element ``all_reduce`` before any other collective, or all
ranks deadlock.
"""
if len(unique_id_bytes) != NCCL_UNIQUE_ID_BYTES:
raise ValueError(
f"expected a {NCCL_UNIQUE_ID_BYTES}-byte NCCL unique id, "
f"got {len(unique_id_bytes)} bytes"
)
if not 0 <= rank < world_size:
raise ValueError(f"rank {rank} out of range for world_size {world_size}")
self = cls.__new__(cls)
self.rank = rank
self.world_size = world_size
self.group = None
if self.world_size == 1 or envs.VLLM_DISABLE_PYNCCL:
self.available = False
self.disabled = True
return self
try:
self.nccl = NCCLLibrary(library_path)
except Exception as e:
# Unlike the TCPStore path, silently disabling here leaves the peer
# blocked in ncclCommInitRank until timeout. The caller explicitly
# asked to join from a unique id, so fail loudly instead.
raise RuntimeError(
"failed to load the NCCL library for unique-id rendezvous"
) from e
self.available = True
self.disabled = False
self.nccl_version = self.nccl.ncclGetRawVersion()
self.unique_id = self.nccl.unique_id_from_bytes(unique_id_bytes)
self._init_comm(device)
return self
def destroy(self):
if self.available and not self.disabled:
# ncclCommAbort can block until all CUDA graphs that
@@ -51,6 +51,9 @@ class ncclUniqueId(ctypes.Structure):
_fields_ = [("internal", ctypes.c_byte * 128)]
NCCL_UNIQUE_ID_BYTES = ctypes.sizeof(ncclUniqueId)
# Mirror of NCCL's versioned ncclCommProperties_t (nccl_device/core.h,
# v2.31.2-1, 144 bytes). ncclCommQueryProperties fills fields gated by the
# version the caller declares in props.version, not by props.size, so never
+125 -8
View File
@@ -8,15 +8,19 @@ The dense (`NCCLWeightTransferEngine`) and sparse
sparse engine does not have to subclass the dense one.
"""
from dataclasses import dataclass
from dataclasses import asdict, dataclass, field
from typing import TYPE_CHECKING, Protocol
import pybase64 as base64
import torch
if TYPE_CHECKING:
from vllm.config.parallel import ParallelConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.device_communicators.pynccl_wrapper import (
NCCL_UNIQUE_ID_BYTES,
)
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
@@ -24,21 +28,102 @@ from vllm.distributed.weight_transfer.packed_tensor import (
)
@dataclass
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
"""Worker-side initialization info for NCCL-based weight transfer backends."""
def decode_nccl_unique_id(
*,
master_address: str | None,
master_port: int | None,
nccl_unique_id_b64: str | None,
ctx: str,
) -> bytes | None:
"""Validate the rendezvous mode and decode a pre-shared unique id."""
has_uid = bool(nccl_unique_id_b64)
has_addr = master_address is not None
has_port = master_port is not None
master_address: str
master_port: int
if has_uid and (has_addr or has_port):
raise ValueError(
f"{ctx}: pass nccl_unique_id_b64 OR master_address/master_port, not both"
)
if not has_uid:
if not (has_addr and has_port):
raise ValueError(
f"{ctx}: need nccl_unique_id_b64, or both master_address and "
f"master_port (got master_address={master_address!r}, "
f"master_port={master_port!r})"
)
return None
try:
decoded = base64.b64decode(nccl_unique_id_b64, validate=True)
except (ValueError, TypeError) as e:
raise ValueError(
f"{ctx}: nccl_unique_id_b64 is not valid standard base64 "
f"(URL-safe base64 is not accepted)"
) from e
if len(decoded) != NCCL_UNIQUE_ID_BYTES:
raise ValueError(
f"{ctx}: nccl_unique_id_b64 must decode to {NCCL_UNIQUE_ID_BYTES} "
f"bytes, got {len(decoded)}"
)
return decoded
@dataclass(kw_only=True)
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
"""Worker-side initialization info for NCCL-based weight transfer backends.
Keyword-only (`kw_only`): adding the optional `nccl_unique_id_b64` field
means the rendezvous fields can no longer keep a fixed positional slot, so a
stale positional call fails loudly instead of silently swapping arguments.
Provide exactly one rendezvous mode:
* ``master_address`` + ``master_port`` -- TCPStore / ``StatelessProcessGroup``
rendezvous (requires torch on every rank, including the trainer), or
* ``nccl_unique_id_b64`` -- standard (RFC 4648, *not* URL-safe) base64 of the
128 raw bytes from ``ncclGetUniqueId``, for torch-free trainers (e.g. JAX)
that mint the unique id out of band and share it (over HTTP, etc.). Note a
JAX peer must use ``base64.b64encode``, not ``urlsafe_b64encode``.
On the unique-id path all ranks must enter init concurrently (there is no
store barrier), and every peer must honor the warm-up handshake: the
worker's communicator issues a one-element ``all_reduce`` immediately after
``ncclCommInitRank`` (see ``PyNcclCommunicator.from_unique_id_bytes``), so a
foreign peer must issue a matching one-element ``all_reduce`` before any
other collective or all ranks deadlock.
"""
master_address: str | None = None
master_port: int | None = None
rank_offset: int
world_size: int
nccl_unique_id_b64: str | None = field(default=None, repr=False)
packed: bool = False
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
def __post_init__(self) -> None:
_ = self.nccl_unique_id_bytes
@property
def nccl_unique_id_bytes(self) -> bytes | None:
return decode_nccl_unique_id(
master_address=self.master_address,
master_port=self.master_port,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="NCCLWeightTransferInitInfo",
)
def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
only the mode actually in use. Shared by the dense and sparse trainer
engines so the two cannot drift."""
return {key: value for key, value in asdict(init_info).items() if value is not None}
class NCCLRendezvous(Protocol):
"""The three fields `trainer_init` needs to open the trainer endpoint.
"""The TCP rendezvous fields `trainer_init` needs.
Structural so each backend can keep its own trainer init info next to its
engine (dense `NCCLTrainerInitInfo`, `SparseNCCLTrainerInitInfo`) without
@@ -73,6 +158,27 @@ def stateless_init_process_group(
return PyNcclCommunicator(pg, device=device)
def uid_init_process_group(
nccl_unique_id_bytes: bytes,
rank: int,
world_size: int,
device,
) -> "PyNcclCommunicator":
"""Join the NCCL group from pre-shared ``ncclUniqueId`` bytes.
The torch-free rendezvous alternative to `stateless_init_process_group`: no
TCPStore, and therefore no barrier -- every rank must enter concurrently.
"""
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
return PyNcclCommunicator.from_unique_id_bytes(
nccl_unique_id_bytes,
rank=rank,
world_size=world_size,
device=device,
)
def worker_init_process_group(
init_info: NCCLWeightTransferInitInfo,
parallel_config: "ParallelConfig",
@@ -80,7 +186,7 @@ def worker_init_process_group(
"""Create the trainer<->worker NCCL group on an inference worker.
Computes a unique rank for this worker across all data-parallel groups and
joins the stateless process group with the trainer.
joins the trainer via whichever rendezvous mode `init_info` carries.
"""
# Calculate the global rank in the trainer-worker process group.
# Must account for data parallel to get unique ranks across all workers.
@@ -93,6 +199,17 @@ def worker_init_process_group(
rank = worker_rank + init_info.rank_offset
device = torch.accelerator.current_device_index()
unique_id_bytes = init_info.nccl_unique_id_bytes
if unique_id_bytes is not None:
return uid_init_process_group(
unique_id_bytes,
rank,
init_info.world_size,
device,
)
# __post_init__ guarantees the TCP pair when there is no unique id.
assert init_info.master_address is not None
assert init_info.master_port is not None
return stateless_init_process_group(
init_info.master_address,
init_info.master_port,
@@ -26,6 +26,7 @@ from vllm.distributed.weight_transfer.base import (
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
worker_init_payload,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_common import (
@@ -57,6 +58,13 @@ class NCCLTrainerInitInfo(TrainerInitInfo):
`world_size` is the full trainer+worker NCCL group size. `rank` (from
`TrainerInitInfo`) identifies this trainer process; rank 0 is the sender.
The trainer joins over a TCPStore rendezvous (`master_address` +
`master_port`). Torch-free trainers (e.g. JAX) that cannot join a TCPStore
mint an `ncclUniqueId` themselves and drive rank 0 out of band; they ship a
`nccl_unique_id_b64` payload straight to the inference workers'
`init_weight_transfer_engine` (see `NCCLWeightTransferInitInfo`) rather than
going through this engine.
`packed` / buffer sizes are the transfer's wire params. The trainer
propagates them to the worker at `trainer_init` so the two sides cannot
disagree. Note this defaults to packed, unlike the worker-side
@@ -301,7 +309,8 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni
# open the trainer endpoint (rank 0); both sides must rendezvous together.
with ThreadPoolExecutor(max_workers=1) as exe:
future = exe.submit(
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
engine.client.init_weight_transfer_engine,
worker_init_payload(worker_init_info),
)
engine.model_update_group = open_trainer_endpoint(init_info)
future.result() # surface any inference-side init error
@@ -31,6 +31,7 @@ from vllm.distributed.weight_transfer.base import (
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
worker_init_payload,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_common import (
@@ -256,7 +257,8 @@ class SparseNCCLTrainerWeightTransferEngine(
# open the trainer endpoint (rank 0); both sides must rendezvous together.
with ThreadPoolExecutor(max_workers=1) as exe:
future = exe.submit(
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
engine.client.init_weight_transfer_engine,
worker_init_payload(worker_init_info),
)
# Open the trainer endpoint as NCCL rank 0 on the current device
# (the init info satisfies the helper's rendezvous protocol).