[Distributed] Support pre-shared ncclUniqueId rendezvous for weight transfer (#53784)
Signed-off-by: Dharak Kharod <dharak.kharod@cohere.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -11,6 +11,7 @@ from torch.distributed import ProcessGroup, ReduceOp
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||
NCCL_UNIQUE_ID_BYTES,
|
||||
NCCLLibrary,
|
||||
buffer_type,
|
||||
cudaStream_t,
|
||||
@@ -60,6 +61,9 @@ def register_nccl_symmetric_ops(pynccl_comm):
|
||||
|
||||
|
||||
class PyNcclCommunicator:
|
||||
# None for communicators built via `from_unique_id_bytes` (no process group).
|
||||
group: ProcessGroup | StatelessProcessGroup | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup | StatelessProcessGroup,
|
||||
@@ -128,6 +132,14 @@ class PyNcclCommunicator:
|
||||
self.unique_id.internal[i] = byte
|
||||
else:
|
||||
self.unique_id = group.broadcast_obj(self.unique_id, src=0)
|
||||
self._init_comm(device)
|
||||
|
||||
def _init_comm(self, device: int | str | torch.device) -> None:
|
||||
"""Create the communicator on `device` from the already-resolved
|
||||
`self.unique_id` / `self.rank` / `self.world_size`, then run the
|
||||
one-element warm-up all_reduce. Shared by `__init__` and
|
||||
`from_unique_id_bytes` so the init handshake stays identical on both.
|
||||
"""
|
||||
if isinstance(device, int):
|
||||
device = torch.device(f"cuda:{device}")
|
||||
elif isinstance(device, str):
|
||||
@@ -148,6 +160,62 @@ class PyNcclCommunicator:
|
||||
stream.synchronize()
|
||||
del data
|
||||
|
||||
@classmethod
|
||||
def from_unique_id_bytes(
|
||||
cls,
|
||||
unique_id_bytes: bytes,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
device: int | str | torch.device,
|
||||
library_path: str | None = None,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Build a communicator from pre-shared ``ncclUniqueId`` bytes.
|
||||
|
||||
For peers that cannot join a ``StatelessProcessGroup`` / TCPStore (e.g. a
|
||||
torch-free JAX trainer): every rank passes the same id, minted once via
|
||||
``ncclGetUniqueId`` and shared out of band. There is no barrier, so all
|
||||
ranks must enter init concurrently or ``ncclCommInitRank`` hangs.
|
||||
|
||||
Warm-up handshake: immediately after ``ncclCommInitRank`` this issues a
|
||||
one-element ``all_reduce`` (mirroring ``__init__``). It is a collective,
|
||||
so every peer -- including a foreign, non-vLLM rank -- must issue a
|
||||
matching one-element ``all_reduce`` before any other collective, or all
|
||||
ranks deadlock.
|
||||
"""
|
||||
if len(unique_id_bytes) != NCCL_UNIQUE_ID_BYTES:
|
||||
raise ValueError(
|
||||
f"expected a {NCCL_UNIQUE_ID_BYTES}-byte NCCL unique id, "
|
||||
f"got {len(unique_id_bytes)} bytes"
|
||||
)
|
||||
if not 0 <= rank < world_size:
|
||||
raise ValueError(f"rank {rank} out of range for world_size {world_size}")
|
||||
|
||||
self = cls.__new__(cls)
|
||||
self.rank = rank
|
||||
self.world_size = world_size
|
||||
self.group = None
|
||||
|
||||
if self.world_size == 1 or envs.VLLM_DISABLE_PYNCCL:
|
||||
self.available = False
|
||||
self.disabled = True
|
||||
return self
|
||||
try:
|
||||
self.nccl = NCCLLibrary(library_path)
|
||||
except Exception as e:
|
||||
# Unlike the TCPStore path, silently disabling here leaves the peer
|
||||
# blocked in ncclCommInitRank until timeout. The caller explicitly
|
||||
# asked to join from a unique id, so fail loudly instead.
|
||||
raise RuntimeError(
|
||||
"failed to load the NCCL library for unique-id rendezvous"
|
||||
) from e
|
||||
|
||||
self.available = True
|
||||
self.disabled = False
|
||||
self.nccl_version = self.nccl.ncclGetRawVersion()
|
||||
self.unique_id = self.nccl.unique_id_from_bytes(unique_id_bytes)
|
||||
self._init_comm(device)
|
||||
return self
|
||||
|
||||
def destroy(self):
|
||||
if self.available and not self.disabled:
|
||||
# ncclCommAbort can block until all CUDA graphs that
|
||||
|
||||
@@ -51,6 +51,9 @@ class ncclUniqueId(ctypes.Structure):
|
||||
_fields_ = [("internal", ctypes.c_byte * 128)]
|
||||
|
||||
|
||||
NCCL_UNIQUE_ID_BYTES = ctypes.sizeof(ncclUniqueId)
|
||||
|
||||
|
||||
# Mirror of NCCL's versioned ncclCommProperties_t (nccl_device/core.h,
|
||||
# v2.31.2-1, 144 bytes). ncclCommQueryProperties fills fields gated by the
|
||||
# version the caller declares in props.version, not by props.size, so never
|
||||
|
||||
Reference in New Issue
Block a user