[Distributed] Support pre-shared ncclUniqueId rendezvous for weight transfer (#53784)
Signed-off-by: Dharak Kharod <dharak.kharod@cohere.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -11,6 +11,7 @@ from torch.distributed import ProcessGroup, ReduceOp
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||
NCCL_UNIQUE_ID_BYTES,
|
||||
NCCLLibrary,
|
||||
buffer_type,
|
||||
cudaStream_t,
|
||||
@@ -60,6 +61,9 @@ def register_nccl_symmetric_ops(pynccl_comm):
|
||||
|
||||
|
||||
class PyNcclCommunicator:
|
||||
# None for communicators built via `from_unique_id_bytes` (no process group).
|
||||
group: ProcessGroup | StatelessProcessGroup | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup | StatelessProcessGroup,
|
||||
@@ -128,6 +132,14 @@ class PyNcclCommunicator:
|
||||
self.unique_id.internal[i] = byte
|
||||
else:
|
||||
self.unique_id = group.broadcast_obj(self.unique_id, src=0)
|
||||
self._init_comm(device)
|
||||
|
||||
def _init_comm(self, device: int | str | torch.device) -> None:
|
||||
"""Create the communicator on `device` from the already-resolved
|
||||
`self.unique_id` / `self.rank` / `self.world_size`, then run the
|
||||
one-element warm-up all_reduce. Shared by `__init__` and
|
||||
`from_unique_id_bytes` so the init handshake stays identical on both.
|
||||
"""
|
||||
if isinstance(device, int):
|
||||
device = torch.device(f"cuda:{device}")
|
||||
elif isinstance(device, str):
|
||||
@@ -148,6 +160,62 @@ class PyNcclCommunicator:
|
||||
stream.synchronize()
|
||||
del data
|
||||
|
||||
@classmethod
|
||||
def from_unique_id_bytes(
|
||||
cls,
|
||||
unique_id_bytes: bytes,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
device: int | str | torch.device,
|
||||
library_path: str | None = None,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Build a communicator from pre-shared ``ncclUniqueId`` bytes.
|
||||
|
||||
For peers that cannot join a ``StatelessProcessGroup`` / TCPStore (e.g. a
|
||||
torch-free JAX trainer): every rank passes the same id, minted once via
|
||||
``ncclGetUniqueId`` and shared out of band. There is no barrier, so all
|
||||
ranks must enter init concurrently or ``ncclCommInitRank`` hangs.
|
||||
|
||||
Warm-up handshake: immediately after ``ncclCommInitRank`` this issues a
|
||||
one-element ``all_reduce`` (mirroring ``__init__``). It is a collective,
|
||||
so every peer -- including a foreign, non-vLLM rank -- must issue a
|
||||
matching one-element ``all_reduce`` before any other collective, or all
|
||||
ranks deadlock.
|
||||
"""
|
||||
if len(unique_id_bytes) != NCCL_UNIQUE_ID_BYTES:
|
||||
raise ValueError(
|
||||
f"expected a {NCCL_UNIQUE_ID_BYTES}-byte NCCL unique id, "
|
||||
f"got {len(unique_id_bytes)} bytes"
|
||||
)
|
||||
if not 0 <= rank < world_size:
|
||||
raise ValueError(f"rank {rank} out of range for world_size {world_size}")
|
||||
|
||||
self = cls.__new__(cls)
|
||||
self.rank = rank
|
||||
self.world_size = world_size
|
||||
self.group = None
|
||||
|
||||
if self.world_size == 1 or envs.VLLM_DISABLE_PYNCCL:
|
||||
self.available = False
|
||||
self.disabled = True
|
||||
return self
|
||||
try:
|
||||
self.nccl = NCCLLibrary(library_path)
|
||||
except Exception as e:
|
||||
# Unlike the TCPStore path, silently disabling here leaves the peer
|
||||
# blocked in ncclCommInitRank until timeout. The caller explicitly
|
||||
# asked to join from a unique id, so fail loudly instead.
|
||||
raise RuntimeError(
|
||||
"failed to load the NCCL library for unique-id rendezvous"
|
||||
) from e
|
||||
|
||||
self.available = True
|
||||
self.disabled = False
|
||||
self.nccl_version = self.nccl.ncclGetRawVersion()
|
||||
self.unique_id = self.nccl.unique_id_from_bytes(unique_id_bytes)
|
||||
self._init_comm(device)
|
||||
return self
|
||||
|
||||
def destroy(self):
|
||||
if self.available and not self.disabled:
|
||||
# ncclCommAbort can block until all CUDA graphs that
|
||||
|
||||
@@ -51,6 +51,9 @@ class ncclUniqueId(ctypes.Structure):
|
||||
_fields_ = [("internal", ctypes.c_byte * 128)]
|
||||
|
||||
|
||||
NCCL_UNIQUE_ID_BYTES = ctypes.sizeof(ncclUniqueId)
|
||||
|
||||
|
||||
# Mirror of NCCL's versioned ncclCommProperties_t (nccl_device/core.h,
|
||||
# v2.31.2-1, 144 bytes). ncclCommQueryProperties fills fields gated by the
|
||||
# version the caller declares in props.version, not by props.size, so never
|
||||
|
||||
@@ -8,15 +8,19 @@ The dense (`NCCLWeightTransferEngine`) and sparse
|
||||
sparse engine does not have to subclass the dense one.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
import pybase64 as base64
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||
NCCL_UNIQUE_ID_BYTES,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
@@ -24,21 +28,102 @@ from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Worker-side initialization info for NCCL-based weight transfer backends."""
|
||||
def decode_nccl_unique_id(
|
||||
*,
|
||||
master_address: str | None,
|
||||
master_port: int | None,
|
||||
nccl_unique_id_b64: str | None,
|
||||
ctx: str,
|
||||
) -> bytes | None:
|
||||
"""Validate the rendezvous mode and decode a pre-shared unique id."""
|
||||
has_uid = bool(nccl_unique_id_b64)
|
||||
has_addr = master_address is not None
|
||||
has_port = master_port is not None
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
if has_uid and (has_addr or has_port):
|
||||
raise ValueError(
|
||||
f"{ctx}: pass nccl_unique_id_b64 OR master_address/master_port, not both"
|
||||
)
|
||||
if not has_uid:
|
||||
if not (has_addr and has_port):
|
||||
raise ValueError(
|
||||
f"{ctx}: need nccl_unique_id_b64, or both master_address and "
|
||||
f"master_port (got master_address={master_address!r}, "
|
||||
f"master_port={master_port!r})"
|
||||
)
|
||||
return None
|
||||
try:
|
||||
decoded = base64.b64decode(nccl_unique_id_b64, validate=True)
|
||||
except (ValueError, TypeError) as e:
|
||||
raise ValueError(
|
||||
f"{ctx}: nccl_unique_id_b64 is not valid standard base64 "
|
||||
f"(URL-safe base64 is not accepted)"
|
||||
) from e
|
||||
if len(decoded) != NCCL_UNIQUE_ID_BYTES:
|
||||
raise ValueError(
|
||||
f"{ctx}: nccl_unique_id_b64 must decode to {NCCL_UNIQUE_ID_BYTES} "
|
||||
f"bytes, got {len(decoded)}"
|
||||
)
|
||||
return decoded
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Worker-side initialization info for NCCL-based weight transfer backends.
|
||||
|
||||
Keyword-only (`kw_only`): adding the optional `nccl_unique_id_b64` field
|
||||
means the rendezvous fields can no longer keep a fixed positional slot, so a
|
||||
stale positional call fails loudly instead of silently swapping arguments.
|
||||
|
||||
Provide exactly one rendezvous mode:
|
||||
|
||||
* ``master_address`` + ``master_port`` -- TCPStore / ``StatelessProcessGroup``
|
||||
rendezvous (requires torch on every rank, including the trainer), or
|
||||
* ``nccl_unique_id_b64`` -- standard (RFC 4648, *not* URL-safe) base64 of the
|
||||
128 raw bytes from ``ncclGetUniqueId``, for torch-free trainers (e.g. JAX)
|
||||
that mint the unique id out of band and share it (over HTTP, etc.). Note a
|
||||
JAX peer must use ``base64.b64encode``, not ``urlsafe_b64encode``.
|
||||
|
||||
On the unique-id path all ranks must enter init concurrently (there is no
|
||||
store barrier), and every peer must honor the warm-up handshake: the
|
||||
worker's communicator issues a one-element ``all_reduce`` immediately after
|
||||
``ncclCommInitRank`` (see ``PyNcclCommunicator.from_unique_id_bytes``), so a
|
||||
foreign peer must issue a matching one-element ``all_reduce`` before any
|
||||
other collective or all ranks deadlock.
|
||||
"""
|
||||
|
||||
master_address: str | None = None
|
||||
master_port: int | None = None
|
||||
rank_offset: int
|
||||
world_size: int
|
||||
nccl_unique_id_b64: str | None = field(default=None, repr=False)
|
||||
packed: bool = False
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_ = self.nccl_unique_id_bytes
|
||||
|
||||
@property
|
||||
def nccl_unique_id_bytes(self) -> bytes | None:
|
||||
return decode_nccl_unique_id(
|
||||
master_address=self.master_address,
|
||||
master_port=self.master_port,
|
||||
nccl_unique_id_b64=self.nccl_unique_id_b64,
|
||||
ctx="NCCLWeightTransferInitInfo",
|
||||
)
|
||||
|
||||
|
||||
def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
|
||||
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
|
||||
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
|
||||
only the mode actually in use. Shared by the dense and sparse trainer
|
||||
engines so the two cannot drift."""
|
||||
return {key: value for key, value in asdict(init_info).items() if value is not None}
|
||||
|
||||
|
||||
class NCCLRendezvous(Protocol):
|
||||
"""The three fields `trainer_init` needs to open the trainer endpoint.
|
||||
"""The TCP rendezvous fields `trainer_init` needs.
|
||||
|
||||
Structural so each backend can keep its own trainer init info next to its
|
||||
engine (dense `NCCLTrainerInitInfo`, `SparseNCCLTrainerInitInfo`) without
|
||||
@@ -73,6 +158,27 @@ def stateless_init_process_group(
|
||||
return PyNcclCommunicator(pg, device=device)
|
||||
|
||||
|
||||
def uid_init_process_group(
|
||||
nccl_unique_id_bytes: bytes,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
device,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Join the NCCL group from pre-shared ``ncclUniqueId`` bytes.
|
||||
|
||||
The torch-free rendezvous alternative to `stateless_init_process_group`: no
|
||||
TCPStore, and therefore no barrier -- every rank must enter concurrently.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
return PyNcclCommunicator.from_unique_id_bytes(
|
||||
nccl_unique_id_bytes,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
def worker_init_process_group(
|
||||
init_info: NCCLWeightTransferInitInfo,
|
||||
parallel_config: "ParallelConfig",
|
||||
@@ -80,7 +186,7 @@ def worker_init_process_group(
|
||||
"""Create the trainer<->worker NCCL group on an inference worker.
|
||||
|
||||
Computes a unique rank for this worker across all data-parallel groups and
|
||||
joins the stateless process group with the trainer.
|
||||
joins the trainer via whichever rendezvous mode `init_info` carries.
|
||||
"""
|
||||
# Calculate the global rank in the trainer-worker process group.
|
||||
# Must account for data parallel to get unique ranks across all workers.
|
||||
@@ -93,6 +199,17 @@ def worker_init_process_group(
|
||||
rank = worker_rank + init_info.rank_offset
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
unique_id_bytes = init_info.nccl_unique_id_bytes
|
||||
if unique_id_bytes is not None:
|
||||
return uid_init_process_group(
|
||||
unique_id_bytes,
|
||||
rank,
|
||||
init_info.world_size,
|
||||
device,
|
||||
)
|
||||
# __post_init__ guarantees the TCP pair when there is no unique id.
|
||||
assert init_info.master_address is not None
|
||||
assert init_info.master_port is not None
|
||||
return stateless_init_process_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.distributed.weight_transfer.base import (
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
worker_init_payload,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
@@ -57,6 +58,13 @@ class NCCLTrainerInitInfo(TrainerInitInfo):
|
||||
`world_size` is the full trainer+worker NCCL group size. `rank` (from
|
||||
`TrainerInitInfo`) identifies this trainer process; rank 0 is the sender.
|
||||
|
||||
The trainer joins over a TCPStore rendezvous (`master_address` +
|
||||
`master_port`). Torch-free trainers (e.g. JAX) that cannot join a TCPStore
|
||||
mint an `ncclUniqueId` themselves and drive rank 0 out of band; they ship a
|
||||
`nccl_unique_id_b64` payload straight to the inference workers'
|
||||
`init_weight_transfer_engine` (see `NCCLWeightTransferInitInfo`) rather than
|
||||
going through this engine.
|
||||
|
||||
`packed` / buffer sizes are the transfer's wire params. The trainer
|
||||
propagates them to the worker at `trainer_init` so the two sides cannot
|
||||
disagree. Note this defaults to packed, unlike the worker-side
|
||||
@@ -301,7 +309,8 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni
|
||||
# open the trainer endpoint (rank 0); both sides must rendezvous together.
|
||||
with ThreadPoolExecutor(max_workers=1) as exe:
|
||||
future = exe.submit(
|
||||
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
|
||||
engine.client.init_weight_transfer_engine,
|
||||
worker_init_payload(worker_init_info),
|
||||
)
|
||||
engine.model_update_group = open_trainer_endpoint(init_info)
|
||||
future.result() # surface any inference-side init error
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.distributed.weight_transfer.base import (
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
worker_init_payload,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
@@ -256,7 +257,8 @@ class SparseNCCLTrainerWeightTransferEngine(
|
||||
# open the trainer endpoint (rank 0); both sides must rendezvous together.
|
||||
with ThreadPoolExecutor(max_workers=1) as exe:
|
||||
future = exe.submit(
|
||||
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
|
||||
engine.client.init_weight_transfer_engine,
|
||||
worker_init_payload(worker_init_info),
|
||||
)
|
||||
# Open the trainer endpoint as NCCL rank 0 on the current device
|
||||
# (the init info satisfies the helper's rendezvous protocol).
|
||||
|
||||
Reference in New Issue
Block a user