[RL] Add sharding-aware NCCL M2N weight transfer (#51520)

Signed-off-by: Ke Wen <kwen@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Ke Wen
2026-09-26 01:38:53 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> aoshen02
parent 4bb804cc5b
commit b6d2b4fc25
13 changed files with 2812 additions and 6 deletions
@@ -243,6 +243,12 @@ WeightTransferEngineFactory.register_engine(
"ShardedRDTWeightTransferEngine",
)
WeightTransferEngineFactory.register_engine(
"nccl_m2n",
"vllm.distributed.weight_transfer.m2n_engine",
"M2NWeightTransferEngine",
)
# Trainer-side engines, parallel to the worker registry above.
WeightTransferTrainerFactory.register_engine(
@@ -268,3 +274,9 @@ WeightTransferTrainerFactory.register_engine(
"vllm.distributed.weight_transfer.sharded_rdt_trainer",
"ShardedRDTTrainerWeightTransferEngine",
)
WeightTransferTrainerFactory.register_engine(
"nccl_m2n",
"vllm.distributed.weight_transfer.m2n_trainer",
"M2NTrainerWeightTransferEngine",
)
@@ -0,0 +1,338 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared helpers for the NCCL M2N (`nccl_m2n`) weight transfer backend.
M2N reshards a tensor between two disjoint meshes of ranks that live in one
communicator: the trainer occupies ranks `[0, T)` and the inference workers
`[T, T + N)`. This module holds everything both sides need — the optional
runtime import, layout descriptors, and the conversion to `nccl.m2n` types —
so the engine module stays about the transfer itself.
"""
from collections.abc import Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
import torch
from vllm.distributed.weight_transfer.base import ParamMeta
if TYPE_CHECKING:
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
# Placement code for a mesh axis that replicates. Any other (non-negative) code
# is the tensor dim that axis shards. Ints rather than `nccl.m2n` objects so
# layouts survive the JSON init handshake without a custom encoder.
REPLICATE = -1
# `NCCL_RESHARD_MAX_TENSOR_DIMS`; 4-D and higher are rejected by the library.
MAX_TENSOR_DIMS = 3
MESH_NDIMS = 2
# Destination mesh convention. Keeping these roles named avoids scattering
# positional axis assumptions through the worker-side planner.
DESTINATION_REPLICA_AXIS = 0
DESTINATION_SHARD_AXIS = 1
# m2n bounds how many source shards may feed one destination shard, and how many
# destination shards one source shard may feed, with compile-time arrays in
# `reshard_limits.h`. The bindings do not expose them, so they are mirrored
# here; a build with larger arrays makes these conservative, never wrong.
MAX_SOURCE_SHARDS = 16
MAX_DEST_SHARDS = 64
# Wire dtypes `ncclReshard` accepts. Notably excludes fp4 and any packed
# sub-byte type, so quantized checkpoints are out of scope for now.
SUPPORTED_DTYPES: frozenset[torch.dtype] = frozenset(
dtype
for dtype in (
getattr(torch, name, None)
for name in (
"int8",
"uint8",
"float8_e4m3fn",
"float8_e5m2",
"float16",
"bfloat16",
"int32",
"uint32",
"float32",
"int64",
"uint64",
"float64",
)
)
if dtype is not None
)
_IMPORT_HINT = (
"The nccl_m2n weight transfer backend requires the `nccl-extensions` "
"package (and its `nccl4py` dependency), which is not installed by vLLM. "
"See https://github.com/NVIDIA/nccl-extensions for build "
"and install instructions. It needs NCCL 2.30.5 or newer, and "
"VLLM_NCCL_SO_PATH must point at the same libnccl.so that libnccl_m2n.so "
"was linked against."
)
def import_m2n() -> Any:
"""Import `nccl.m2n` lazily, with an actionable error when it is missing.
Deferred so that importing vLLM — or any other weight transfer backend —
never requires the m2n runtime to be present.
"""
try:
import nccl.m2n as m2n
except ImportError as e:
raise ImportError(f"{_IMPORT_HINT} (import failed: {e})") from e
return m2n
@dataclass(frozen=True)
class M2NMesh:
"""One side's rank topology — pure topology, no tensor placement.
Mirrors `ncclMesh_t`: a 2-axis mesh owning the contiguous rank interval
`[start_rank, start_rank + dims[0] * dims[1])`. There is no 1-D mesh; a
single-axis topology is spelled with a second axis of size 1.
One mesh describes every tensor on its side, so it is exchanged once at the
init handshake rather than per parameter.
"""
dims: tuple[int, int]
start_rank: int
def __post_init__(self) -> None:
if len(self.dims) != MESH_NDIMS or any(d <= 0 for d in self.dims):
raise ValueError(f"mesh dims must be {MESH_NDIMS} positive ints")
if self.start_rank < 0:
raise ValueError(
f"mesh start_rank must be non-negative, got {self.start_rank}"
)
@property
def size(self) -> int:
return self.dims[0] * self.dims[1]
# A tensor's placement over its side's mesh: one code per mesh axis
# (`REPLICATE`, or the tensor dim that axis shards), or `REPLICATED` for a
# tensor every rank holds in full.
Placements = tuple[int, int]
REPLICATED: Placements | None = None
def check_placements(placements: Placements, context: str = "placements") -> None:
"""Reject placement pairs m2n cannot express.
The header requires exactly one SHARD axis and one REPLICATE axis.
`{REPLICATE, REPLICATE}` hits a degenerate prepare branch, which is why
full replication is carried as `REPLICATED` and resolved separately;
sharding both axes is not expressible at all.
"""
if len(placements) != MESH_NDIMS:
raise ValueError(f"{context} must have {MESH_NDIMS} entries")
invalid = [code for code in placements if code < REPLICATE]
if invalid:
raise ValueError(
f"{context} contains invalid placement code {invalid[0]}; valid "
f"codes are {REPLICATE} (Replicate) and non-negative tensor dimensions"
)
num_sharded = sum(code != REPLICATE for code in placements)
if num_sharded == 0:
raise ValueError(
f"{context}: a fully replicated tensor is carried as REPLICATED, "
"not as two "
"REPLICATE axes"
)
if num_sharded == MESH_NDIMS:
raise ValueError(
f"{context}: nccl_m2n needs one REPLICATE mesh axis, but "
f"{placements} shards both"
)
def resolve_layout(
mesh: M2NMesh,
placements: Placements | None,
context: str = "placements",
) -> tuple[M2NMesh, Placements]:
"""Pair one tensor's placement with the mesh m2n should see for it.
A replicated tensor needs a size-1 mesh axis to carry a no-op shard, since
m2n has no `{REPLICATE, REPLICATE}`. It is therefore described over the
*same rank interval* re-factored as `(size, 1)`. That re-factoring is sound
precisely because replication is order-independent — every rank holds the
whole tensor, so it does not matter that `(a, b)` and `(size, 1)` walk the
interval in a different order. A sharded tensor keeps its side's own
factorization, where rank order decides who owns which shard.
"""
if placements is None:
return M2NMesh((mesh.size, 1), mesh.start_rank), (REPLICATE, 0)
check_placements(placements, context)
return mesh, placements
def shard_count(mesh: M2NMesh, placements: Placements) -> int:
"""How many pieces this layout splits the tensor into."""
return next(
(mesh.dims[axis] for axis, code in enumerate(placements) if code != REPLICATE),
1,
)
def check_plan_limits(
src: tuple[M2NMesh, Placements],
dst: tuple[M2NMesh, Placements],
name: str,
) -> None:
"""Reject plans that exceed m2n's static per-shard fan-in / fan-out arrays.
Only the unambiguous cases are checked: when one side is a single shard it
is fed by (or feeds) every shard on the other side, so the count is exactly
the other side's shard count. In the general sharded-to-sharded case the
overlap depends on the library's chunking, and reproducing that arithmetic
here would duplicate internals that can change under us.
m2n re-checks authoritatively and, because the plan is derived from the
shared descriptors, fails identically on every rank -- so this is about
reporting at init with a message that names the parameter, not about
avoiding a hang.
"""
src_shards = shard_count(*src)
dst_shards = shard_count(*dst)
if dst_shards == 1 and src_shards > MAX_SOURCE_SHARDS:
raise ValueError(
f"parameter '{name}' would feed {src_shards} source shards into one "
f"replicated destination, over m2n's MAX_SOURCES={MAX_SOURCE_SHARDS}. "
"Shard the destination, replicate the source, or rebuild m2n with "
"larger arrays."
)
if src_shards == 1 and dst_shards > MAX_DEST_SHARDS:
raise ValueError(
f"parameter '{name}' would fan one source shard out to {dst_shards} "
f"destination shards, over m2n's MAX_TARGETS={MAX_DEST_SHARDS}. "
"Rebuild m2n with larger arrays to raise the bound."
)
def validate_layout(
mesh: M2NMesh, placements: Placements, shape: Sequence[int], side: str
) -> None:
"""Check a resolved layout can describe `shape` before any collective runs.
Called on both sides at init so a bad plan surfaces as an error from the
init RPC rather than as a hang inside the first reshard.
"""
for axis, code in enumerate(placements):
if code == REPLICATE:
continue
if code >= len(shape):
raise ValueError(
f"{side} layout shards tensor dim {code}, but the tensor "
f"has rank {len(shape)}"
)
factor = mesh.dims[axis]
if shape[code] % factor:
raise ValueError(
f"{side} layout shards dim {code} (size {shape[code]}) over "
f"{factor} ranks, which does not divide evenly"
)
@dataclass(frozen=True)
class M2NParamMeta(ParamMeta):
"""`ParamMeta` extended with how the trainer places this tensor.
The base class carries only name / dtype / full shape, which is not enough
to plan a reshard. `placements` is relative to its side's `M2NMesh`, or
`REPLICATED` when every rank holds the whole tensor.
"""
placements: Placements | None
def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> None:
"""Reject tensors m2n cannot move at all, with the parameter named."""
if dtype not in SUPPORTED_DTYPES:
raise ValueError(
f"parameter '{name}' has dtype {dtype}, which nccl_m2n does not "
f"support. Supported: {sorted(str(d) for d in SUPPORTED_DTYPES)}"
)
if not 1 <= len(shape) <= MAX_TENSOR_DIMS:
raise ValueError(
f"parameter '{name}' has rank {len(shape)}; nccl_m2n supports "
f"rank 1..{MAX_TENSOR_DIMS}"
)
def to_mesh(m2n: Any, mesh: M2NMesh) -> Any:
return m2n.Mesh(mesh.dims, start_rank=mesh.start_rank)
def to_placements(m2n: Any, placements: Placements) -> list[Any]:
return [
m2n.Replicate() if code == REPLICATE else m2n.Shard(code) for code in placements
]
def publish_destination_placements(
comm: "PyNcclCommunicator",
first_worker_rank: int,
placements: "Sequence[Placements | None] | None",
num_parameters: int,
) -> list[Placements | None]:
"""Share the worker-side destination plan with every rank in the group.
The trainer must issue each reshard with the same destination the workers
use, but once destinations are per-parameter they depend on the inference
model, which only the workers can see. The first worker publishes them here,
over the shared NCCL communicator; trainer ranks pass `None` and receive
them, and the other workers pass their own so a disagreement is caught
rather than deadlocking later.
Only placements travel: the destination *mesh* is still derived from the
rank split, identically on both sides. Use the NCCL communicator directly
because unique-id rendezvous deliberately has no bootstrap process group.
"""
replicated_sentinel = REPLICATE - 1
encoded = torch.full(
(num_parameters, MESH_NDIMS),
replicated_sentinel,
dtype=torch.int8,
device=comm.device,
)
if comm.rank == first_worker_rank:
if placements is None or len(placements) != num_parameters:
raise ValueError(
f"publishing rank needs {num_parameters} destination placements"
)
rows = [
[replicated_sentinel] * MESH_NDIMS if placement is REPLICATED else placement
for placement in placements
]
if rows:
encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=comm.device))
comm.broadcast(encoded, src=first_worker_rank)
return [
REPLICATED
if all(code == replicated_sentinel for code in row)
else cast(Placements, tuple(row))
for row in encoded.tolist()
]
def comm_ptr(comm: "PyNcclCommunicator") -> int:
"""Raw `ncclComm_t` behind vLLM's `PyNcclCommunicator`.
m2n links its own NCCL, so the handle only means anything if vLLM loaded
the same `libnccl.so` — set `VLLM_NCCL_SO_PATH` accordingly.
"""
handle = comm.comm
ptr = getattr(handle, "value", handle)
if not ptr:
raise RuntimeError("PyNcclCommunicator has no live NCCL communicator")
return int(ptr)
@@ -0,0 +1,453 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-side weight transfer engine built on NCCL M2N (`nccl_m2n`).
The trainer and the inference workers share one NCCL communicator: trainer ranks
occupy `[0, T)`, workers `[T, T + N)`. Each parameter is moved with a single
`nccl.m2n.reshard`, which redistributes it from the trainer's layout (FSDP / EP
/ arbitrary DTensor sharding) to the inference layout — so the trainer sends its
local shards and never all-gathers a full tensor, which is what the broadcast
NCCL backend forces it to do.
When an incoming checkpoint parameter maps directly to a live vLLM parameter,
M2N reshards into that worker's local model storage. Parameters whose names or
layouts cannot be resolved receive a full tensor and fall back to
`load_weights`, preserving the existing backend's loading behavior.
Both meshes participate in every reshard, in the same order, so this backend has
the same concurrency shape as the broadcast NCCL backend: the worker must be
inside `receive_weights` while the trainer is sending. Driving that from the
trainer is the trainer engine's job.
"""
from collections.abc import Iterator
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, cast
import torch
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferEngine,
WeightTransferInitInfo,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.m2n_common import (
DESTINATION_SHARD_AXIS,
MESH_NDIMS,
REPLICATED,
M2NMesh,
M2NParamMeta,
Placements,
check_plan_limits,
check_transferable,
comm_ptr,
import_m2n,
publish_destination_placements,
resolve_layout,
to_mesh,
to_placements,
validate_layout,
)
from vllm.distributed.weight_transfer.m2n_layout import (
M2NDestination,
resolve_parameter_destinations,
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
decode_nccl_unique_id,
worker_init_process_group,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
__all__ = [
"M2NWeightTransferInitInfo",
"M2NWeightTransferUpdateInfo",
"M2NWeightTransferEngine",
]
# ---------------------------------------------------------------------------
# Wire types
# ---------------------------------------------------------------------------
@dataclass(kw_only=True)
class M2NWeightTransferInitInfo(WeightTransferInitInfo):
"""Worker-side init info: the rendezvous plus the full transfer plan.
Layouts are static for the whole run, so they ride the one-time init
handshake and the per-round update info stays a list of names. Everything is
plain JSON so the HTTP control plane carries it unchanged.
"""
master_address: str | None = None
master_port: int | None = None
rank_offset: int
"""First worker rank, i.e. the number of trainer ranks."""
world_size: int
"""Trainer ranks + all inference workers."""
src_mesh_dims: list[int]
"""The trainer's mesh, shared by every parameter (`start_rank` is 0: the
trainer occupies the front of the communicator)."""
dst_mesh_dims: list[int]
"""The inference mesh, starting at `rank_offset`. Declared rather than
derived so both sides describe the destination identically."""
names: list[str]
dtype_names: list[str]
shapes: list[list[int]]
src_placements: list[list[int] | None]
"""Per parameter, relative to `src_mesh_dims`; `None` means replicated."""
nccl_unique_id_b64: str | None = field(default=None, repr=False)
max_cta: int | None = None
def __post_init__(self) -> None:
_ = self.nccl_unique_id_bytes
num_params = len(self.names)
for label, values in (
("dtype_names", self.dtype_names),
("shapes", self.shapes),
("src_placements", self.src_placements),
):
if len(values) != num_params:
raise ValueError(
f"`{label}` should be of the same size as `names`: "
f"got {len(values)} and {num_params}"
)
if self.rank_offset < 1 or self.rank_offset >= self.world_size:
raise ValueError(
f"`rank_offset` ({self.rank_offset}) must leave at least one "
f"trainer rank and one worker in world_size {self.world_size}"
)
num_workers = self.world_size - self.rank_offset
dst = tuple(self.dst_mesh_dims)
if len(dst) != MESH_NDIMS or dst[0] * dst[1] != num_workers:
raise ValueError(
f"`dst_mesh_dims` {dst} must be {MESH_NDIMS} dims covering the "
f"{num_workers} inference workers"
)
@property
def nccl_unique_id_bytes(self) -> bytes | None:
return decode_nccl_unique_id(
master_address=self.master_address,
master_port=self.master_port,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="M2NWeightTransferInitInfo",
)
@dataclass
class M2NWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Per-round update info: which parameters this chunk carries, in order.
Shapes, dtypes and layouts were fixed at init, so a round only needs to say
what is coming and in what order — both sides must issue their reshards in
exactly that order.
"""
names: list[str]
# ---------------------------------------------------------------------------
# Worker engine
# ---------------------------------------------------------------------------
class M2NWeightTransferEngine(
WeightTransferEngine[M2NWeightTransferInitInfo, M2NWeightTransferUpdateInfo]
):
"""Inference-side engine: receives each parameter with one reshard.
Resolvable parameters are resharded from the trainer layout directly into
each worker's live local model storage. Unresolvable parameters receive a
full tensor and use `load_weights`. In both cases, the trainer sends its
local shards without materializing a full tensor.
"""
init_info_cls = M2NWeightTransferInitInfo
update_info_cls = M2NWeightTransferUpdateInfo
# The destination plan is resolved against the model this engine was built
# for, so it cannot be retargeted at a draft model mid-flight.
supports_draft_weight_update = False
def __init__(
self,
config: WeightTransferConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
super().__init__(config, vllm_config, device, model)
self.model_update_group: PyNcclCommunicator | None = None
self._m2n: Any = None
self._handle: Any = None
self._metas: list[M2NParamMeta] = []
self._src_mesh: M2NMesh | None = None
self._dst_mesh: M2NMesh | None = None
self._parameter_destinations: list[M2NDestination] = []
self._index: dict[str, int] = {}
self._uses_load_weights = False
def init_transfer_engine(self, init_info: M2NWeightTransferInitInfo) -> None:
"""Join the trainer's communicator and rebuild the transfer plan.
Every precondition (dtype, tensor rank, divisibility) is checked here so
a bad plan fails the init RPC instead of hanging the first collective.
"""
self._m2n = import_m2n()
self._metas = []
self._src_mesh = M2NMesh(
cast(tuple[int, int], tuple(init_info.src_mesh_dims)), 0
)
if self._src_mesh.size != init_info.rank_offset:
raise ValueError(
f"source mesh covers {self._src_mesh.size} ranks, but there are "
f"{init_info.rank_offset} trainer ranks"
)
# The inference topology, as declared by the trainer. This is the mesh
# a sharded destination is placed over; a replicated one is described
# by the alternate descriptor `resolve_layout` derives from it, which
# covers the same ranks but is not this mesh.
self._dst_mesh = M2NMesh(
cast(tuple[int, int], tuple(init_info.dst_mesh_dims)),
init_info.rank_offset,
)
for name, name_dtype, shape, placements in zip(
init_info.names,
init_info.dtype_names,
init_info.shapes,
init_info.src_placements,
):
dtype = getattr(torch, name_dtype, None)
if not isinstance(dtype, torch.dtype):
raise ValueError(
f"parameter '{name}' has invalid dtype name "
f"'{name_dtype}'; expected the name of a torch.dtype"
)
check_transferable(name, dtype, shape)
codes = (
REPLICATED
if placements is None
else cast(Placements, tuple(placements))
)
src_mesh, src_placements = resolve_layout(
self._src_mesh, codes, f"parameter '{name}' source placements"
)
validate_layout(src_mesh, src_placements, shape, "source")
# Destination checks are deferred until loader-aware resolution;
# assuming replication here can reject a valid sharded plan.
self._metas.append(M2NParamMeta(name, dtype, tuple(shape), codes))
parallel_config = self.parallel_config
shard_axis_size = self._dst_mesh.dims[DESTINATION_SHARD_AXIS]
quantization = getattr(self.model_config, "quantization", None)
# A quantized weight loader may dequantize / transpose / requantize, and
# under PP a rank's local shape is not the checkpoint shape split over
# the shard axis. Both break shape-derived resolution, so take the
# fallback.
allow_direct = (
quantization is None and parallel_config.pipeline_parallel_size == 1
)
# Match every incoming checkpoint parameter to its rank-local target.
# Resolvable parameters can be resharded directly into live model
# storage; the rest receive a full tensor for `load_weights`.
self._parameter_destinations = resolve_parameter_destinations(
self.model,
[m.name for m in self._metas],
[m.dtype for m in self._metas],
[m.shape for m in self._metas],
num_workers=self._dst_mesh.size,
shard_axis_size=shard_axis_size,
allow_direct=allow_direct,
)
# Check the resolved layout rather than a provisional replicated one.
# For example, 32 source shards feeding 4 destination shards may need
# only 8 sources per destination, while a replicated plan needs all 32.
for meta, destination in zip(self._metas, self._parameter_destinations):
src_mesh, src_placements = resolve_layout(
self._src_mesh,
meta.placements,
f"parameter '{meta.name}' source placements",
)
dst_mesh, dst_placements = resolve_layout(
self._dst_mesh,
destination.placements,
f"parameter '{meta.name}' destination placements",
)
validate_layout(dst_mesh, dst_placements, meta.shape, "destination")
check_plan_limits(
(src_mesh, src_placements),
(dst_mesh, dst_placements),
meta.name,
)
# Update requests carry names only, so cache their plan indices. Track
# whether any fallback entry requires the `load_weights` lifecycle.
self._index = {meta.name: i for i, meta in enumerate(self._metas)}
self._uses_load_weights = any(
not destination.direct for destination in self._parameter_destinations
)
self.model_update_group = worker_init_process_group(
NCCLWeightTransferInitInfo(
master_address=init_info.master_address,
master_port=init_info.master_port,
nccl_unique_id_b64=init_info.nccl_unique_id_b64,
rank_offset=init_info.rank_offset,
world_size=init_info.world_size,
),
parallel_config,
)
# Destinations are per-parameter and depend on the inference model, so
# the trainer cannot derive them. The first worker publishes the plan
# and every other rank checks its own against it — a silent
# disagreement between workers would mean mismatched collectives.
mine = [destination.placements for destination in self._parameter_destinations]
agreed = publish_destination_placements(
self.model_update_group, init_info.rank_offset, mine, len(self._metas)
)
if agreed != mine:
mismatched = next(
meta.name for meta, a, b in zip(self._metas, mine, agreed) if a != b
)
raise RuntimeError(
"inference workers resolved different nccl_m2n destinations "
f"(first mismatch: '{mismatched}'); all workers must run the "
"same model and parallel config"
)
self._handle = self._m2n.Handle.create(
self._m2n.Config(max_cta=init_info.max_cta)
)
def start_weight_update(self) -> None:
"""Set up layerwise reloading, but only if some parameter needs it.
Directly-resharded parameters are written in place and never go through
`load_weights`, so a plan with no fallback entries has nothing to
reload.
"""
if not self._uses_load_weights:
return
from vllm.model_executor.model_loader.reload import initialize_layerwise_reload
initialize_layerwise_reload(self.model)
def finish_weight_update(self) -> None:
"""Finalize layerwise reloading when the plan uses fallback entries."""
if not self._uses_load_weights:
return
from vllm.model_executor.model_loader.reload import finalize_layerwise_reload
finalize_layerwise_reload(self.model, self.model_config)
def receive_weights(self, update_info: M2NWeightTransferUpdateInfo) -> None:
"""Receive each requested parameter using its initialization-time plan."""
if self._handle is None or self.model_update_group is None:
raise RuntimeError(
"nccl_m2n weight transfer not initialized. "
"Call init_transfer_engine() first."
)
requested = []
for name in update_info.names:
index = self._index.get(name)
if index is None:
raise ValueError(
f"parameter '{name}' was not declared at init; the "
"trainer must send the same parameter set it announced"
)
requested.append((name, index))
from vllm.model_executor.model_loader.mtp_validation import (
disable_mtp_completeness_check,
)
comm = comm_ptr(self.model_update_group)
stream = torch.cuda.current_stream()
# Reshard lazily so one `load_weights` invocation sees the complete
# fallback sequence without staging every full tensor at once. Direct
# destinations still execute in request order as the iterator advances.
def reshard_requested_weights() -> Iterator[tuple[str, torch.Tensor]]:
for name, index in requested:
meta = self._metas[index]
destination = self._parameter_destinations[index]
buffer = destination.tensor
if buffer is None:
buffer = torch.empty(
meta.shape, dtype=meta.dtype, device=self.device
)
self._reshard(comm, stream, meta, destination.placements, buffer)
if destination.tensor is None:
# `load_weights` reads on the host stream, so the transfer
# has to have landed before it runs.
stream.synchronize()
yield name, buffer
has_fallback = any(
self._parameter_destinations[index].tensor is None for _, index in requested
)
with disable_mtp_completeness_check():
received_weights = reshard_requested_weights()
if has_fallback:
# Some model loaders keep invocation-local state to combine
# related checkpoint tensors, so preserve one iterable scope.
self.model.load_weights(received_weights)
else:
# The iterator performs the reshards. With nothing to yield to
# `load_weights`, exhaust it here to execute direct transfers.
for _ in received_weights:
pass
def _reshard(
self,
comm: int,
stream: torch.cuda.Stream,
src_meta: M2NParamMeta,
dst_placements: Placements | None,
dst_buffer: torch.Tensor,
) -> None:
"""Reshard one parameter from its trainer layout into a worker buffer.
`dst_placements` describes the buffer's logical placement over the
worker mesh; `dst_buffer` is this rank's physical storage.
"""
assert self._src_mesh is not None
assert self._dst_mesh is not None
m2n = self._m2n
src_mesh, resolved_src_placements = resolve_layout(
self._src_mesh, src_meta.placements
)
dst_mesh, resolved_dst_placements = resolve_layout(
self._dst_mesh, dst_placements
)
m2n.reshard(
None,
dst_buffer,
comm,
stream,
src_mesh=to_mesh(m2n, src_mesh),
src_placements=to_placements(m2n, resolved_src_placements),
dst_mesh=to_mesh(m2n, dst_mesh),
dst_placements=to_placements(m2n, resolved_dst_placements),
handle=self._handle,
)
def shutdown(self) -> None:
"""Finish pending GPU work and release M2N communication state."""
if self._handle is not None:
# M2N does not synchronize caller streams on finalize.
torch.accelerator.synchronize()
self._handle.destroy()
self._handle = None
self.model_update_group = None
self._parameter_destinations = []
@@ -0,0 +1,261 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Destination-layout resolution for the NCCL M2N weight transfer backend.
For every incoming checkpoint parameter the worker needs a destination buffer
plus its placement over the inference mesh. Two outcomes are possible:
* **direct** — when the model is not quantized, the parameter maps 1:1 onto a
live vLLM parameter and its loader is a small, known-safe copy or tensor-
parallel loader. The loader's declared input/output dimension determines the
placement; shapes are validation, not an inference mechanism. The reshard
writes straight into the live parameter, so each rank receives only its own
shard and nothing is copied afterwards.
* **fallback** — anything else. The reshard delivers the whole tensor to every
rank and `load_weights` does the sharding, exactly as the broadcast NCCL
backend does. Fused parameters (`qkv_proj`, `gate_up_proj`, MoE `w13`/`w2`)
take this path: the checkpoint name does not name a vLLM parameter, so there
is nothing to resolve against.
Correctness never depends on a parameter resolving — the fallback is always
available and is the same path the existing backend uses.
"""
import inspect
import math
from collections.abc import Sequence
from dataclasses import dataclass
from functools import partial
from typing import cast
import torch
from vllm.distributed.weight_transfer.m2n_common import (
DESTINATION_REPLICA_AXIS,
DESTINATION_SHARD_AXIS,
MESH_NDIMS,
REPLICATE,
REPLICATED,
Placements,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
_DIRECT_WEIGHT_LOADING_BLOCKLIST = {
"vllm.model_executor.models.gpt2.GPT2Model",
}
def _module_type_name(module: torch.nn.Module) -> str:
cls = type(module)
return f"{cls.__module__}.{cls.__name__}"
def _destination_placements(shard_dim: int) -> Placements:
"""Place one tensor shard on the configured destination mesh axis."""
placements = [REPLICATE] * MESH_NDIMS
placements[DESTINATION_REPLICA_AXIS] = REPLICATE
placements[DESTINATION_SHARD_AXIS] = shard_dim
return cast(Placements, tuple(placements))
@dataclass
class M2NDestination:
"""Where one checkpoint parameter lands on this worker."""
name: str
placements: Placements | None
"""Placement over the inference mesh, or `REPLICATED` for the fallback."""
tensor: torch.Tensor | None
"""The live parameter view to reshard into, or None for the fallback path
(the engine allocates a full replica per round and calls `load_weights`)."""
@property
def direct(self) -> bool:
return self.tensor is not None
def _base_loader_shard_dim(
param: torch.nn.Parameter,
shard_axis_size: int,
) -> int | None:
"""Return a known-safe loader's declared shard dim, or reject it.
Loader identity is deliberately fail-closed. A missing loader is not
treated as ``default_weight_loader`` because a model-level ``load_weights``
implementation may transform the tensor before it reaches that default.
Wrappers and partials are also rejected: recognizing their wrapped callable
would ignore behavior added by the wrapper.
"""
loader = getattr(param, "weight_loader", None)
if loader is None or isinstance(loader, partial):
return None
owner = loader.__self__ if inspect.ismethod(loader) else None
loader_fn = loader.__func__ if inspect.ismethod(loader) else loader
# Lazy imports avoid making the weight-transfer registry import all model
# executor layers during normal vLLM startup.
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
replicated_loaders = {
default_weight_loader,
ReplicatedLinear.weight_loader,
}
column_loaders = {
ColumnParallelLinear.weight_loader,
ColumnParallelLinear.weight_loader_v2,
}
row_loaders = {
RowParallelLinear.weight_loader,
RowParallelLinear.weight_loader_v2,
}
# These attributes signal packing, a transform, or pre-sharded checkpoint
# storage. Such semantics cannot be reproduced by a plain M2N placement.
# Check them before accepting replicated loaders too: model-level loading
# may transform a tensor before handing it to an otherwise plain copier.
metadata_attrs = ("packed_dim", "packed_factor", "marlin_tile_size")
flag_attrs = ("needs_scalar_to_array", "is_sharded_weight", "is_transposed")
if any(getattr(param, attr, None) is not None for attr in metadata_attrs):
return None
if any(bool(getattr(param, attr, False)) for attr in flag_attrs):
return None
if loader_fn in replicated_loaders:
return REPLICATE
if loader_fn not in column_loaders | row_loaders:
return None
if owner is None or getattr(owner, "tp_size", None) != shard_axis_size:
return None
attr = "output_dim" if loader_fn in column_loaders else "input_dim"
dim = getattr(param, attr, None)
return dim if isinstance(dim, int) and dim >= 0 else None
def _validated_shard_dim(
global_shape: Sequence[int],
local_shape: Sequence[int],
declared_dim: int,
shard_axis_size: int,
) -> int | None:
"""Validate a loader-declared shard dimension against both shapes."""
if len(global_shape) != len(local_shape):
return None
if declared_dim == REPLICATE:
return REPLICATE if tuple(global_shape) == tuple(local_shape) else None
if declared_dim >= len(global_shape):
return None
for dim, (whole, local) in enumerate(zip(global_shape, local_shape)):
expected = local * shard_axis_size if dim == declared_dim else local
if whole != expected:
return None
return declared_dim
def resolve_parameter_destinations(
model: torch.nn.Module,
names: Sequence[str],
dtypes: Sequence[torch.dtype],
shapes: Sequence[Sequence[int]],
*,
num_workers: int,
shard_axis_size: int,
allow_direct: bool,
) -> list[M2NDestination]:
"""Build this worker's destination plan, one entry per checkpoint parameter.
Placements are relative to the inference mesh: axis 0 replicates and axis 1
shards. `allow_direct=False` forces every parameter onto the fallback path;
the engine sets it for pipeline-parallel or quantized deployments, where a
parameter's local shape is not simply the checkpoint shape split across the
shard axis. Parameters below a model type with known model-level checkpoint
preprocessing also fall back, since bypassing `load_weights` would skip the
transformation.
"""
params = dict(model.named_parameters()) if allow_direct else {}
blocked_param_ids = {
id(param)
for module in model.modules()
if _module_type_name(module) in _DIRECT_WEIGHT_LOADING_BLOCKLIST
for param in module.parameters()
}
destinations: list[M2NDestination] = []
for name, dtype, shape in zip(names, dtypes, shapes):
param = params.get(name)
dim = None
if (
param is not None
and id(param) not in blocked_param_ids
and param.dtype == dtype
and param.data.is_contiguous()
and num_workers % shard_axis_size == 0
):
declared_dim = _base_loader_shard_dim(param, shard_axis_size)
if declared_dim is not None:
dim = _validated_shard_dim(
shape, param.shape, declared_dim, shard_axis_size
)
if dim is None:
# A replicated parameter is identical on every rank, so it needs no
# placement of its own — REPLICATED lets resolve_layout spread it
# over all the workers regardless of how the mesh is factored.
destinations.append(M2NDestination(name, REPLICATED, None))
elif dim == REPLICATE:
assert param is not None
destinations.append(M2NDestination(name, REPLICATED, param.data))
else:
assert param is not None
destinations.append(
M2NDestination(name, _destination_placements(dim), param.data)
)
# Layerwise reload finalizes a module as a unit. Mixing direct and fallback
# parameters within one module can copy stale storage over the direct load.
fallback_modules = {
name.rpartition(".")[0]
for name, destination in zip(names, destinations)
if not destination.direct and name in params
}
destinations = [
M2NDestination(destination.name, REPLICATED, None)
if destination.direct
and destination.name.rpartition(".")[0] in fallback_modules
else destination
for destination in destinations
]
num_direct = sum(d.direct for d in destinations)
parameter_bytes = [
math.prod(shape) * dtype.itemsize for dtype, shape in zip(dtypes, shapes)
]
total_bytes = sum(parameter_bytes)
direct_bytes = sum(
nbytes
for destination, nbytes in zip(destinations, parameter_bytes)
if destination.direct
)
direct_byte_percentage = 100 * direct_bytes / total_bytes if total_bytes else 0
logger.info(
"nccl_m2n destination plan: %d/%d parameters resharded directly into "
"the model, %d via full-tensor fallback; direct byte coverage: "
"%d/%d bytes (%.1f%%)",
num_direct,
len(destinations),
len(destinations) - num_direct,
direct_bytes,
total_bytes,
direct_byte_percentage,
)
return destinations
@@ -0,0 +1,155 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Trainer-side weight sources for the NCCL M2N backend.
m2n plans a transfer from both sides' layouts, so the trainer has to say how it
holds each parameter — which the base `WeightSource` / `ParamMeta` pair does not
express. This module supplies the m2n flavor of both, and the DTensor-backed
implementation that covers the common trainer.
A source declares its mesh once (`mesh()`) and a placement per parameter, since
one topology describes every tensor on the side.
"""
from collections.abc import Iterator
from typing import Any
import torch
from vllm.distributed.weight_transfer.base import ParamMeta, WeightSource
from vllm.distributed.weight_transfer.m2n_common import (
REPLICATE,
REPLICATED,
M2NMesh,
M2NParamMeta,
Placements,
)
__all__ = [
"M2NWeightSource",
"DTensorModuleSource",
"mesh_from_tensor",
"placements_from_tensor",
]
class M2NWeightSource(WeightSource):
"""A `WeightSource` that also describes how the trainer holds its weights.
Unlike `ModuleSource`, iteration yields each rank's **local shard**, not a
materialized full tensor: gathering is exactly the cost m2n removes.
"""
def mesh(self) -> M2NMesh:
"""The trainer's rank topology, shared by every parameter."""
raise NotImplementedError
def metadata(self) -> list[ParamMeta]:
"""Name, dtype, full shape, and trainer placement for each parameter."""
raise NotImplementedError
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
"""Yield `(name, local shard)` pairs in the same order as `metadata()`."""
raise NotImplementedError
def _placement_code(placement: Any) -> int:
"""Map a `torch.distributed` placement onto an m2n placement code."""
name = type(placement).__name__
if name == "Replicate":
return REPLICATE
if name == "Shard":
return int(placement.dim)
raise ValueError(
f"nccl_m2n cannot express the {name} placement; only Replicate and "
"Shard are supported"
)
def mesh_from_tensor(tensor: torch.Tensor, num_trainer_ranks: int) -> M2NMesh:
"""The mesh a parameter lives on, as an `M2NMesh`.
A plain tensor is identical on every trainer rank, so it spans all of them.
"""
device_mesh = getattr(tensor, "device_mesh", None)
if device_mesh is None:
return M2NMesh((num_trainer_ranks, 1), 0)
grid = device_mesh.mesh
ranks = grid.flatten().tolist()
if ranks != list(range(len(ranks))):
raise ValueError(
"nccl_m2n requires the trainer's device mesh to cover the "
f"contiguous rank interval [0, {len(ranks)}); got {ranks}"
)
if grid.ndim > 2:
raise ValueError(
f"nccl_m2n supports 1-D and 2-D device meshes, got {grid.ndim}-D"
)
dims = tuple(grid.shape)
return M2NMesh(dims if len(dims) == 2 else (dims[0], 1), 0)
def placements_from_tensor(tensor: torch.Tensor) -> Placements | None:
"""How a parameter is placed over its mesh, or `REPLICATED`.
`REPLICATED` is returned rather than a `(REPLICATE, REPLICATE)` pair: m2n
cannot express that, and the size-1-axis encoding it needs instead is
applied later by `resolve_layout`, which owns that workaround.
"""
placements = getattr(tensor, "placements", None)
if placements is None:
return REPLICATED
codes = [_placement_code(p) for p in placements]
if all(code == REPLICATE for code in codes):
return REPLICATED
if len(codes) == 1:
return (codes[0], REPLICATE)
return (codes[0], codes[1])
class DTensorModuleSource(M2NWeightSource):
"""`M2NWeightSource` over `module.named_parameters()`.
Covers both the FSDP/DTensor trainer (placement read off each parameter,
local shard yielded via `to_local()`) and the plain replicated trainer, with
no special casing. Trainers with a custom producer (a Megatron export, MoE
re-fusing) subclass `M2NWeightSource` instead.
"""
def __init__(self, module: torch.nn.Module, num_trainer_ranks: int) -> None:
"""Wrap `module`; `num_trainer_ranks` sizes the mesh for plain tensors."""
self._module = module
self._num_trainer_ranks = num_trainer_ranks
def mesh(self) -> M2NMesh:
"""The mesh every sharded parameter agrees on.
Replicated parameters span the trainer without a device mesh of their
own, so they never decide this; a model whose *sharded* parameters
disagree cannot be described by one side mesh and is rejected.
"""
meshes = {
mesh_from_tensor(p, self._num_trainer_ranks)
for _, p in self._module.named_parameters()
if placements_from_tensor(p) is not REPLICATED
}
if len(meshes) > 1:
raise ValueError(
"nccl_m2n needs one mesh per side, but this module's sharded "
f"parameters span several: {sorted(m.dims for m in meshes)}"
)
return meshes.pop() if meshes else M2NMesh((self._num_trainer_ranks, 1), 0)
def metadata(self) -> list[ParamMeta]:
"""Read global shape/dtype and placements without gathering shards."""
return [
M2NParamMeta(name, p.dtype, tuple(p.shape), placements_from_tensor(p))
for name, p in self._module.named_parameters()
]
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
"""Yield each parameter's local shard (`to_local()`), or the tensor itself."""
for name, param in self._module.named_parameters():
to_local = getattr(param, "to_local", None)
yield name, (to_local() if callable(to_local) else param)
@@ -0,0 +1,414 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Trainer-side weight transfer engine for the NCCL M2N backend.
Symmetric to `M2NWeightTransferEngine` but in the training process. Every
trainer rank joins the shared communicator and runs every reshard, sending its
own local shard; only rank 0 touches the inference control plane.
Workers join the communicator during init and run reshard from inside
`update_weights`, so those two RPCs overlap with work on this side. Both run on
a helper thread and are joined afterwards, which keeps that overlap inside the
engine -- where the `TrainerWeightTransferEngine` contract puts it -- rather than
in every caller.
"""
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar, cast
import torch
from typing_extensions import Self
from vllm.distributed.weight_transfer.base import (
TrainerInitInfo,
TrainerWeightTransferEngine,
VLLMWeightSyncClient,
WeightSource,
)
from vllm.distributed.weight_transfer.m2n_common import (
M2NMesh,
M2NParamMeta,
Placements,
check_transferable,
comm_ptr,
import_m2n,
publish_destination_placements,
resolve_layout,
to_mesh,
to_placements,
validate_layout,
)
from vllm.distributed.weight_transfer.m2n_source import M2NWeightSource
from vllm.distributed.weight_transfer.nccl_common import (
decode_nccl_unique_id,
trainer_init,
uid_init_process_group,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
logger = init_logger(__name__)
__all__ = ["M2NTrainerInitInfo", "M2NTrainerWeightTransferEngine"]
def _dtype_name(dtype: torch.dtype) -> str:
"""Short name for a dtype, e.g. `bfloat16`, for the worker-side init dict."""
return str(dtype).split(".")[-1]
@dataclass(kw_only=True)
class M2NTrainerInitInfo(TrainerInitInfo):
"""Trainer-side init info for nccl_m2n.
`rank` (from `TrainerInitInfo`) is this trainer process's rank; it is also
its rank in the shared communicator, since the trainer occupies
`[0, num_trainer_ranks)`. Rank 0 drives the control plane.
"""
backend: ClassVar[str] = "nccl_m2n"
master_address: str | None = None
master_port: int | None = None
world_size: int
"""Trainer ranks + all inference workers."""
num_trainer_ranks: int = 1
dst_mesh_dims: tuple[int, int] | None = None
"""How the inference ranks are laid out: axis 0 replicates and axis 1
shards. The trainer declares it so both sides describe the destination
identically; defaults to a flat `(num_workers, 1)`, which is all a
replicated destination needs."""
max_cta: int | None = None
nccl_unique_id_b64: str | None = field(default=None, repr=False)
@property
def destination_mesh_dims(self) -> tuple[int, int]:
"""`dst_mesh_dims`, or a flat mesh over every inference worker."""
if self.dst_mesh_dims is not None:
return self.dst_mesh_dims
return (self.world_size - self.num_trainer_ranks, 1)
def __post_init__(self) -> None:
"""Reject rank counts or destination meshes that cannot form a group."""
_ = self.nccl_unique_id_bytes
if not 0 < self.num_trainer_ranks < self.world_size:
raise ValueError(
f"`num_trainer_ranks` ({self.num_trainer_ranks}) must leave at "
f"least one worker in world_size {self.world_size}"
)
num_workers = self.world_size - self.num_trainer_ranks
if self.dst_mesh_dims is not None:
dims = tuple(self.dst_mesh_dims)
if len(dims) != 2 or dims[0] * dims[1] != num_workers:
raise ValueError(
f"`dst_mesh_dims` {dims} must be two dims covering the "
f"{num_workers} inference workers"
)
if not 0 <= self.rank < self.num_trainer_ranks:
raise ValueError(
f"trainer `rank` ({self.rank}) must be below "
f"`num_trainer_ranks` ({self.num_trainer_ranks})"
)
@property
def nccl_unique_id_bytes(self) -> bytes | None:
return decode_nccl_unique_id(
master_address=self.master_address,
master_port=self.master_port,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="M2NTrainerInitInfo",
)
class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitInfo]):
"""Trainer-side engine: sends every rank's local shard, once per parameter.
Called on every trainer rank. All ranks join the communicator and run every
reshard; only rank 0 touches the control plane.
"""
init_info_cls = M2NTrainerInitInfo
def __init__(
self,
*,
client: VLLMWeightSyncClient,
source: M2NWeightSource,
is_controller: bool,
num_trainer_ranks: int,
) -> None:
"""Hold the client and source; `trainer_init` fills in the group."""
super().__init__(client=client, source=source, is_sender=is_controller)
self.source: M2NWeightSource = source
self.is_controller = is_controller
self.num_trainer_ranks = num_trainer_ranks
self.group: PyNcclCommunicator | None = None
self._m2n: Any = None
self._handle: Any = None
self._metas: list[M2NParamMeta] = []
self._src_mesh: M2NMesh | None = None
self._dst_mesh: M2NMesh | None = None
self._dst_placements: list[Placements | None] = []
self._executor: ThreadPoolExecutor | None = None
@classmethod
def trainer_init(
cls,
init_info: M2NTrainerInitInfo,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
) -> Self:
"""Build the engine and rendezvous with the inference side.
Runs on *every* trainer rank. Rank 0 additionally ships the transfer
plan to the workers and drives the control plane; the other ranks only
build local state and join the communicator. Every rank participates
in every reshard and sends its local shard.
The ordering here is not arbitrary -- see the comments inline. In
short: validate before anything blocks; start the init RPC without
waiting (workers join the communicator inside that RPC, so it cannot
return until this side has joined too); then join the NCCL
communicator; then create the M2N handle; and only then wait for the RPC
to finish.
"""
# The base WeightSource carries name/dtype/shape but not layout, which
# is not enough to plan a reshard. Fail here rather than at the first
# missing attribute deep inside the send loop.
if not isinstance(source, M2NWeightSource):
raise TypeError(
"nccl_m2n needs per-parameter layouts, so its source must be an "
f"M2NWeightSource (e.g. DTensorModuleSource); got "
f"{type(source).__name__}"
)
# Trainer-side weight transfer engine for this rank. `__init__` only
# holds the client and source; the communicator, meshes, and M2N handle
# are filled in below.
engine = cls(
client=client,
source=source,
is_controller=init_info.rank == 0,
num_trainer_ranks=init_info.num_trainer_ranks,
)
engine._m2n = import_m2n()
engine._src_mesh = source.mesh()
if engine._src_mesh.size != init_info.num_trainer_ranks:
raise ValueError(
f"source mesh covers {engine._src_mesh.size} ranks, but there "
f"are {init_info.num_trainer_ranks} trainer ranks"
)
# Validate the whole plan before any rendezvous. A layout m2n cannot
# express is a mismatched collective at send time, which hangs rather
# than raises -- so every precondition is checked while an exception can
# still propagate to the caller.
engine._metas = cast(list[M2NParamMeta], source.metadata())
for meta in engine._metas:
check_transferable(meta.name, meta.dtype, meta.shape)
mesh, placements = resolve_layout(engine._src_mesh, meta.placements)
validate_layout(mesh, placements, meta.shape, "source")
rendezvous: Future | None = None
if engine.is_controller:
engine._executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="m2n-weight-sync"
)
# Only trainer rank 0 starts the control-plane RPC. Workers join
# the communicator inside it, so submit it first, join below, and
# wait for it at the end.
rendezvous = engine._executor.submit(
client.init_weight_transfer_engine,
engine._worker_init_info(init_info),
)
# Every trainer rank joins -- reshard is collective over the whole
# communicator. Trainer ranks take
# [0, num_trainer_ranks) and the workers follow, which is the
# contiguous-interval layout m2n requires of both meshes.
unique_id_bytes = init_info.nccl_unique_id_bytes
if unique_id_bytes is not None:
engine.group = uid_init_process_group(
unique_id_bytes,
rank=init_info.rank,
world_size=init_info.world_size,
device=torch.accelerator.current_device_index(),
)
else:
assert init_info.master_address is not None
assert init_info.master_port is not None
engine.group = trainer_init(
{
"master_address": init_info.master_address,
"master_port": init_info.master_port,
"world_size": init_info.world_size,
},
rank=init_info.rank,
)
# The trainer declares the inference topology; the worker uses exactly
# this, so neither side has to infer the other's factorization.
engine._dst_mesh = M2NMesh(
init_info.destination_mesh_dims, init_info.num_trainer_ranks
)
engine._dst_placements = engine._receive_destination_plan(
init_info.num_trainer_ranks
)
logger.info(
"nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), "
"inference ranks [%d, %d)",
len(engine._metas),
init_info.num_trainer_ranks,
init_info.num_trainer_ranks,
init_info.world_size,
)
# Created after the group so a failed rendezvous does not leave a live
# handle behind; shutdown() releases it once caller streams are synced.
engine._handle = engine._m2n.Handle.create(
engine._m2n.Config(max_cta=init_info.max_cta)
)
# Now safe to collect: both sides have joined, so the RPC has returned
# or raised. Doing this last means a worker-side init failure surfaces
# here, as an exception from trainer_init, rather than at the first send.
if rendezvous is not None:
rendezvous.result()
return engine
def _receive_destination_plan(
self, first_worker_rank: int
) -> list[Placements | None]:
"""Receive and validate the inference-side per-parameter placements."""
if self.group is None or self._dst_mesh is None:
raise RuntimeError(
"destination plan requires the communicator and destination mesh"
)
# The first inference worker publishes one placement per parameter.
# Every trainer rank receives the complete plan in this single,
# initialization-time broadcast.
placements = publish_destination_placements(
self.group, first_worker_rank, None, len(self._metas)
)
# The send loop matches placements to metadata by position, so reject
# an incomplete or oversized plan before entering any M2N collective.
if len(placements) != len(self._metas):
raise RuntimeError(
f"inference side planned {len(placements)} parameters, but "
f"this trainer announced {len(self._metas)}"
)
# Check every worker-selected destination against the corresponding
# global parameter shape before the first transfer.
for meta, placement in zip(self._metas, placements):
mesh, resolved = resolve_layout(self._dst_mesh, placement)
validate_layout(mesh, resolved, meta.shape, "destination")
return placements
def _worker_init_info(self, init_info: M2NTrainerInitInfo) -> dict[str, Any]:
"""Handshake payload: rendezvous, both meshes, and the transfer plan."""
assert self._src_mesh is not None
payload: dict[str, Any] = {
"master_address": init_info.master_address,
"master_port": init_info.master_port,
"nccl_unique_id_b64": init_info.nccl_unique_id_b64,
"rank_offset": init_info.num_trainer_ranks,
"world_size": init_info.world_size,
"src_mesh_dims": list(self._src_mesh.dims),
"dst_mesh_dims": list(init_info.destination_mesh_dims),
"names": [m.name for m in self._metas],
"dtype_names": [_dtype_name(m.dtype) for m in self._metas],
"shapes": [list(m.shape) for m in self._metas],
"src_placements": [
None if m.placements is None else list(m.placements)
for m in self._metas
],
"max_cta": init_info.max_cta,
}
return {key: value for key, value in payload.items() if value is not None}
def send_weights(self) -> None:
"""Drive one update round: start, reshard concurrently, then finish."""
if self._handle is None or self.group is None:
raise RuntimeError("nccl_m2n trainer engine is not initialized")
update: Future | None = None
if self.is_controller:
self.client.start_weight_update()
# The workers reshard from inside `update_weights`, concurrently
# with the sends below.
assert self._executor is not None
update = self._executor.submit(
self.client.update_weights,
{"names": [m.name for m in self._metas]},
)
try:
self._send()
finally:
if update is not None:
update.result()
if self.is_controller:
self.client.finish_weight_update()
def _send(self) -> None:
"""Reshard each local shard into its worker-planned destination."""
assert self.group is not None
assert self._src_mesh is not None
assert self._dst_mesh is not None
m2n = self._m2n
comm = comm_ptr(self.group)
stream = torch.cuda.current_stream()
for index, (name, tensor) in enumerate(self.source):
meta = self._metas[index]
if name != meta.name:
raise RuntimeError(
f"weight source yielded '{name}' at position {index}, but "
f"its metadata declared '{meta.name}'; the source must "
"iterate in a stable order"
)
shard = tensor.detach().contiguous()
src_mesh, src_placements = resolve_layout(self._src_mesh, meta.placements)
dst_mesh, dst_placements = resolve_layout(
self._dst_mesh, self._dst_placements[index]
)
m2n.reshard(
shard,
None,
comm,
stream,
src_mesh=to_mesh(m2n, src_mesh),
src_placements=to_placements(m2n, src_placements),
dst_mesh=to_mesh(m2n, dst_mesh),
dst_placements=to_placements(m2n, dst_placements),
handle=self._handle,
)
# A custom source may yield temporary storage allocated on another
# stream. M2N retains only its device pointer for asynchronous work,
# so tell the allocator not to reuse that storage until this stream
# has finished with it. For storage allocated on this stream,
# record_stream is effectively a no-op.
shard.record_stream(stream)
# As in the other weight-transfer backends, wait before returning
# rather than relying on same-stream ordering. The caller may then
# safely mutate parameters or continue work on another stream.
stream.synchronize()
def shutdown(self) -> None:
"""Destroy the m2n handle, join the helper thread, and drop the group."""
if self._handle is not None:
torch.accelerator.synchronize()
self._handle.destroy()
self._handle = None
if self._executor is not None:
self._executor.shutdown()
self._executor = None
self.group = None
@@ -220,11 +220,11 @@ def worker_init_process_group(
def trainer_init(
init_info: NCCLRendezvous | dict,
rank: int = 0,
) -> "PyNcclCommunicator":
"""Initialize NCCL process group for trainer-side weight transfer.
The trainer is always rank 0 in the process group. Uses the current
CUDA device (torch.accelerator.current_device_index()).
Uses the current CUDA device (torch.accelerator.current_device_index()).
Args:
init_info: Any object carrying the `NCCLRendezvous` fields (a trainer or
@@ -232,6 +232,9 @@ def trainer_init(
- master_address: str
- master_port: int
- world_size: int
rank: This trainer process's rank in the group. The broadcast backends
have a single trainer at rank 0; multi-rank trainers (m2n) occupy
ranks `[0, num_trainer_ranks)` and the workers start after them.
Returns:
PyNcclCommunicator for weight transfer.
@@ -246,12 +249,11 @@ def trainer_init(
master_port = init_info.master_port
world_size = init_info.world_size
# Trainer is always rank 0
device = torch.accelerator.current_device_index()
return stateless_init_process_group(
master_address,
master_port,
0,
rank,
world_size,
device,
)