[Distributed] Support pre-shared ncclUniqueId rendezvous for weight transfer (#53784)

Signed-off-by: Dharak Kharod <dharak.kharod@cohere.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
dharak-cohere
2026-09-01 14:55:57 +08:00
committed by GitHub
co-authored by Cursor
parent fa99a6fea6
commit d0e695a91b
6 changed files with 674 additions and 10 deletions
+125 -8
View File
@@ -8,15 +8,19 @@ The dense (`NCCLWeightTransferEngine`) and sparse
sparse engine does not have to subclass the dense one.
"""
from dataclasses import dataclass
from dataclasses import asdict, dataclass, field
from typing import TYPE_CHECKING, Protocol
import pybase64 as base64
import torch
if TYPE_CHECKING:
from vllm.config.parallel import ParallelConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.device_communicators.pynccl_wrapper import (
NCCL_UNIQUE_ID_BYTES,
)
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
@@ -24,21 +28,102 @@ from vllm.distributed.weight_transfer.packed_tensor import (
)
@dataclass
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
"""Worker-side initialization info for NCCL-based weight transfer backends."""
def decode_nccl_unique_id(
*,
master_address: str | None,
master_port: int | None,
nccl_unique_id_b64: str | None,
ctx: str,
) -> bytes | None:
"""Validate the rendezvous mode and decode a pre-shared unique id."""
has_uid = bool(nccl_unique_id_b64)
has_addr = master_address is not None
has_port = master_port is not None
master_address: str
master_port: int
if has_uid and (has_addr or has_port):
raise ValueError(
f"{ctx}: pass nccl_unique_id_b64 OR master_address/master_port, not both"
)
if not has_uid:
if not (has_addr and has_port):
raise ValueError(
f"{ctx}: need nccl_unique_id_b64, or both master_address and "
f"master_port (got master_address={master_address!r}, "
f"master_port={master_port!r})"
)
return None
try:
decoded = base64.b64decode(nccl_unique_id_b64, validate=True)
except (ValueError, TypeError) as e:
raise ValueError(
f"{ctx}: nccl_unique_id_b64 is not valid standard base64 "
f"(URL-safe base64 is not accepted)"
) from e
if len(decoded) != NCCL_UNIQUE_ID_BYTES:
raise ValueError(
f"{ctx}: nccl_unique_id_b64 must decode to {NCCL_UNIQUE_ID_BYTES} "
f"bytes, got {len(decoded)}"
)
return decoded
@dataclass(kw_only=True)
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
"""Worker-side initialization info for NCCL-based weight transfer backends.
Keyword-only (`kw_only`): adding the optional `nccl_unique_id_b64` field
means the rendezvous fields can no longer keep a fixed positional slot, so a
stale positional call fails loudly instead of silently swapping arguments.
Provide exactly one rendezvous mode:
* ``master_address`` + ``master_port`` -- TCPStore / ``StatelessProcessGroup``
rendezvous (requires torch on every rank, including the trainer), or
* ``nccl_unique_id_b64`` -- standard (RFC 4648, *not* URL-safe) base64 of the
128 raw bytes from ``ncclGetUniqueId``, for torch-free trainers (e.g. JAX)
that mint the unique id out of band and share it (over HTTP, etc.). Note a
JAX peer must use ``base64.b64encode``, not ``urlsafe_b64encode``.
On the unique-id path all ranks must enter init concurrently (there is no
store barrier), and every peer must honor the warm-up handshake: the
worker's communicator issues a one-element ``all_reduce`` immediately after
``ncclCommInitRank`` (see ``PyNcclCommunicator.from_unique_id_bytes``), so a
foreign peer must issue a matching one-element ``all_reduce`` before any
other collective or all ranks deadlock.
"""
master_address: str | None = None
master_port: int | None = None
rank_offset: int
world_size: int
nccl_unique_id_b64: str | None = field(default=None, repr=False)
packed: bool = False
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
def __post_init__(self) -> None:
_ = self.nccl_unique_id_bytes
@property
def nccl_unique_id_bytes(self) -> bytes | None:
return decode_nccl_unique_id(
master_address=self.master_address,
master_port=self.master_port,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="NCCLWeightTransferInitInfo",
)
def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
only the mode actually in use. Shared by the dense and sparse trainer
engines so the two cannot drift."""
return {key: value for key, value in asdict(init_info).items() if value is not None}
class NCCLRendezvous(Protocol):
"""The three fields `trainer_init` needs to open the trainer endpoint.
"""The TCP rendezvous fields `trainer_init` needs.
Structural so each backend can keep its own trainer init info next to its
engine (dense `NCCLTrainerInitInfo`, `SparseNCCLTrainerInitInfo`) without
@@ -73,6 +158,27 @@ def stateless_init_process_group(
return PyNcclCommunicator(pg, device=device)
def uid_init_process_group(
nccl_unique_id_bytes: bytes,
rank: int,
world_size: int,
device,
) -> "PyNcclCommunicator":
"""Join the NCCL group from pre-shared ``ncclUniqueId`` bytes.
The torch-free rendezvous alternative to `stateless_init_process_group`: no
TCPStore, and therefore no barrier -- every rank must enter concurrently.
"""
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
return PyNcclCommunicator.from_unique_id_bytes(
nccl_unique_id_bytes,
rank=rank,
world_size=world_size,
device=device,
)
def worker_init_process_group(
init_info: NCCLWeightTransferInitInfo,
parallel_config: "ParallelConfig",
@@ -80,7 +186,7 @@ def worker_init_process_group(
"""Create the trainer<->worker NCCL group on an inference worker.
Computes a unique rank for this worker across all data-parallel groups and
joins the stateless process group with the trainer.
joins the trainer via whichever rendezvous mode `init_info` carries.
"""
# Calculate the global rank in the trainer-worker process group.
# Must account for data parallel to get unique ranks across all workers.
@@ -93,6 +199,17 @@ def worker_init_process_group(
rank = worker_rank + init_info.rank_offset
device = torch.accelerator.current_device_index()
unique_id_bytes = init_info.nccl_unique_id_bytes
if unique_id_bytes is not None:
return uid_init_process_group(
unique_id_bytes,
rank,
init_info.world_size,
device,
)
# __post_init__ guarantees the TCP pair when there is no unique id.
assert init_info.master_address is not None
assert init_info.master_port is not None
return stateless_init_process_group(
init_info.master_address,
init_info.master_port,
@@ -26,6 +26,7 @@ from vllm.distributed.weight_transfer.base import (
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
worker_init_payload,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_common import (
@@ -57,6 +58,13 @@ class NCCLTrainerInitInfo(TrainerInitInfo):
`world_size` is the full trainer+worker NCCL group size. `rank` (from
`TrainerInitInfo`) identifies this trainer process; rank 0 is the sender.
The trainer joins over a TCPStore rendezvous (`master_address` +
`master_port`). Torch-free trainers (e.g. JAX) that cannot join a TCPStore
mint an `ncclUniqueId` themselves and drive rank 0 out of band; they ship a
`nccl_unique_id_b64` payload straight to the inference workers'
`init_weight_transfer_engine` (see `NCCLWeightTransferInitInfo`) rather than
going through this engine.
`packed` / buffer sizes are the transfer's wire params. The trainer
propagates them to the worker at `trainer_init` so the two sides cannot
disagree. Note this defaults to packed, unlike the worker-side
@@ -301,7 +309,8 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni
# open the trainer endpoint (rank 0); both sides must rendezvous together.
with ThreadPoolExecutor(max_workers=1) as exe:
future = exe.submit(
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
engine.client.init_weight_transfer_engine,
worker_init_payload(worker_init_info),
)
engine.model_update_group = open_trainer_endpoint(init_info)
future.result() # surface any inference-side init error
@@ -31,6 +31,7 @@ from vllm.distributed.weight_transfer.base import (
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
worker_init_payload,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_common import (
@@ -256,7 +257,8 @@ class SparseNCCLTrainerWeightTransferEngine(
# open the trainer endpoint (rank 0); both sides must rendezvous together.
with ThreadPoolExecutor(max_workers=1) as exe:
future = exe.submit(
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
engine.client.init_weight_transfer_engine,
worker_init_payload(worker_init_info),
)
# Open the trainer endpoint as NCCL rank 0 on the current device
# (the init info satisfies the helper's rendezvous protocol).