[Distributed] Support pre-shared ncclUniqueId rendezvous for weight transfer (#53784)

Signed-off-by: Dharak Kharod <dharak.kharod@cohere.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
dharak-cohere
2026-09-01 14:55:57 +08:00
committed by GitHub
co-authored by Cursor
parent fa99a6fea6
commit d0e695a91b
6 changed files with 674 additions and 10 deletions
@@ -11,6 +11,7 @@ from torch.distributed import ProcessGroup, ReduceOp
import vllm.envs as envs
from vllm.distributed.device_communicators.pynccl_wrapper import (
NCCL_UNIQUE_ID_BYTES,
NCCLLibrary,
buffer_type,
cudaStream_t,
@@ -60,6 +61,9 @@ def register_nccl_symmetric_ops(pynccl_comm):
class PyNcclCommunicator:
# None for communicators built via `from_unique_id_bytes` (no process group).
group: ProcessGroup | StatelessProcessGroup | None
def __init__(
self,
group: ProcessGroup | StatelessProcessGroup,
@@ -128,6 +132,14 @@ class PyNcclCommunicator:
self.unique_id.internal[i] = byte
else:
self.unique_id = group.broadcast_obj(self.unique_id, src=0)
self._init_comm(device)
def _init_comm(self, device: int | str | torch.device) -> None:
"""Create the communicator on `device` from the already-resolved
`self.unique_id` / `self.rank` / `self.world_size`, then run the
one-element warm-up all_reduce. Shared by `__init__` and
`from_unique_id_bytes` so the init handshake stays identical on both.
"""
if isinstance(device, int):
device = torch.device(f"cuda:{device}")
elif isinstance(device, str):
@@ -148,6 +160,62 @@ class PyNcclCommunicator:
stream.synchronize()
del data
@classmethod
def from_unique_id_bytes(
cls,
unique_id_bytes: bytes,
rank: int,
world_size: int,
device: int | str | torch.device,
library_path: str | None = None,
) -> "PyNcclCommunicator":
"""Build a communicator from pre-shared ``ncclUniqueId`` bytes.
For peers that cannot join a ``StatelessProcessGroup`` / TCPStore (e.g. a
torch-free JAX trainer): every rank passes the same id, minted once via
``ncclGetUniqueId`` and shared out of band. There is no barrier, so all
ranks must enter init concurrently or ``ncclCommInitRank`` hangs.
Warm-up handshake: immediately after ``ncclCommInitRank`` this issues a
one-element ``all_reduce`` (mirroring ``__init__``). It is a collective,
so every peer -- including a foreign, non-vLLM rank -- must issue a
matching one-element ``all_reduce`` before any other collective, or all
ranks deadlock.
"""
if len(unique_id_bytes) != NCCL_UNIQUE_ID_BYTES:
raise ValueError(
f"expected a {NCCL_UNIQUE_ID_BYTES}-byte NCCL unique id, "
f"got {len(unique_id_bytes)} bytes"
)
if not 0 <= rank < world_size:
raise ValueError(f"rank {rank} out of range for world_size {world_size}")
self = cls.__new__(cls)
self.rank = rank
self.world_size = world_size
self.group = None
if self.world_size == 1 or envs.VLLM_DISABLE_PYNCCL:
self.available = False
self.disabled = True
return self
try:
self.nccl = NCCLLibrary(library_path)
except Exception as e:
# Unlike the TCPStore path, silently disabling here leaves the peer
# blocked in ncclCommInitRank until timeout. The caller explicitly
# asked to join from a unique id, so fail loudly instead.
raise RuntimeError(
"failed to load the NCCL library for unique-id rendezvous"
) from e
self.available = True
self.disabled = False
self.nccl_version = self.nccl.ncclGetRawVersion()
self.unique_id = self.nccl.unique_id_from_bytes(unique_id_bytes)
self._init_comm(device)
return self
def destroy(self):
if self.available and not self.disabled:
# ncclCommAbort can block until all CUDA graphs that
@@ -51,6 +51,9 @@ class ncclUniqueId(ctypes.Structure):
_fields_ = [("internal", ctypes.c_byte * 128)]
NCCL_UNIQUE_ID_BYTES = ctypes.sizeof(ncclUniqueId)
# Mirror of NCCL's versioned ncclCommProperties_t (nccl_device/core.h,
# v2.31.2-1, 144 bytes). ncclCommQueryProperties fills fields gated by the
# version the caller declares in props.version, not by props.size, so never