[RL] Add sharding-aware NCCL M2N weight transfer (#51520)
Signed-off-by: Ke Wen <kwen@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
aoshen02
parent
4bb804cc5b
commit
b6d2b4fc25
@@ -243,6 +243,12 @@ WeightTransferEngineFactory.register_engine(
|
||||
"ShardedRDTWeightTransferEngine",
|
||||
)
|
||||
|
||||
WeightTransferEngineFactory.register_engine(
|
||||
"nccl_m2n",
|
||||
"vllm.distributed.weight_transfer.m2n_engine",
|
||||
"M2NWeightTransferEngine",
|
||||
)
|
||||
|
||||
|
||||
# Trainer-side engines, parallel to the worker registry above.
|
||||
WeightTransferTrainerFactory.register_engine(
|
||||
@@ -268,3 +274,9 @@ WeightTransferTrainerFactory.register_engine(
|
||||
"vllm.distributed.weight_transfer.sharded_rdt_trainer",
|
||||
"ShardedRDTTrainerWeightTransferEngine",
|
||||
)
|
||||
|
||||
WeightTransferTrainerFactory.register_engine(
|
||||
"nccl_m2n",
|
||||
"vllm.distributed.weight_transfer.m2n_trainer",
|
||||
"M2NTrainerWeightTransferEngine",
|
||||
)
|
||||
|
||||
@@ -0,0 +1,338 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared helpers for the NCCL M2N (`nccl_m2n`) weight transfer backend.
|
||||
|
||||
M2N reshards a tensor between two disjoint meshes of ranks that live in one
|
||||
communicator: the trainer occupies ranks `[0, T)` and the inference workers
|
||||
`[T, T + N)`. This module holds everything both sides need — the optional
|
||||
runtime import, layout descriptors, and the conversion to `nccl.m2n` types —
|
||||
so the engine module stays about the transfer itself.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.base import ParamMeta
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
# Placement code for a mesh axis that replicates. Any other (non-negative) code
|
||||
# is the tensor dim that axis shards. Ints rather than `nccl.m2n` objects so
|
||||
# layouts survive the JSON init handshake without a custom encoder.
|
||||
REPLICATE = -1
|
||||
|
||||
# `NCCL_RESHARD_MAX_TENSOR_DIMS`; 4-D and higher are rejected by the library.
|
||||
MAX_TENSOR_DIMS = 3
|
||||
|
||||
MESH_NDIMS = 2
|
||||
|
||||
# Destination mesh convention. Keeping these roles named avoids scattering
|
||||
# positional axis assumptions through the worker-side planner.
|
||||
DESTINATION_REPLICA_AXIS = 0
|
||||
DESTINATION_SHARD_AXIS = 1
|
||||
|
||||
# m2n bounds how many source shards may feed one destination shard, and how many
|
||||
# destination shards one source shard may feed, with compile-time arrays in
|
||||
# `reshard_limits.h`. The bindings do not expose them, so they are mirrored
|
||||
# here; a build with larger arrays makes these conservative, never wrong.
|
||||
MAX_SOURCE_SHARDS = 16
|
||||
MAX_DEST_SHARDS = 64
|
||||
|
||||
# Wire dtypes `ncclReshard` accepts. Notably excludes fp4 and any packed
|
||||
# sub-byte type, so quantized checkpoints are out of scope for now.
|
||||
SUPPORTED_DTYPES: frozenset[torch.dtype] = frozenset(
|
||||
dtype
|
||||
for dtype in (
|
||||
getattr(torch, name, None)
|
||||
for name in (
|
||||
"int8",
|
||||
"uint8",
|
||||
"float8_e4m3fn",
|
||||
"float8_e5m2",
|
||||
"float16",
|
||||
"bfloat16",
|
||||
"int32",
|
||||
"uint32",
|
||||
"float32",
|
||||
"int64",
|
||||
"uint64",
|
||||
"float64",
|
||||
)
|
||||
)
|
||||
if dtype is not None
|
||||
)
|
||||
|
||||
_IMPORT_HINT = (
|
||||
"The nccl_m2n weight transfer backend requires the `nccl-extensions` "
|
||||
"package (and its `nccl4py` dependency), which is not installed by vLLM. "
|
||||
"See https://github.com/NVIDIA/nccl-extensions for build "
|
||||
"and install instructions. It needs NCCL 2.30.5 or newer, and "
|
||||
"VLLM_NCCL_SO_PATH must point at the same libnccl.so that libnccl_m2n.so "
|
||||
"was linked against."
|
||||
)
|
||||
|
||||
|
||||
def import_m2n() -> Any:
|
||||
"""Import `nccl.m2n` lazily, with an actionable error when it is missing.
|
||||
|
||||
Deferred so that importing vLLM — or any other weight transfer backend —
|
||||
never requires the m2n runtime to be present.
|
||||
"""
|
||||
try:
|
||||
import nccl.m2n as m2n
|
||||
except ImportError as e:
|
||||
raise ImportError(f"{_IMPORT_HINT} (import failed: {e})") from e
|
||||
return m2n
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NMesh:
|
||||
"""One side's rank topology — pure topology, no tensor placement.
|
||||
|
||||
Mirrors `ncclMesh_t`: a 2-axis mesh owning the contiguous rank interval
|
||||
`[start_rank, start_rank + dims[0] * dims[1])`. There is no 1-D mesh; a
|
||||
single-axis topology is spelled with a second axis of size 1.
|
||||
|
||||
One mesh describes every tensor on its side, so it is exchanged once at the
|
||||
init handshake rather than per parameter.
|
||||
"""
|
||||
|
||||
dims: tuple[int, int]
|
||||
start_rank: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if len(self.dims) != MESH_NDIMS or any(d <= 0 for d in self.dims):
|
||||
raise ValueError(f"mesh dims must be {MESH_NDIMS} positive ints")
|
||||
if self.start_rank < 0:
|
||||
raise ValueError(
|
||||
f"mesh start_rank must be non-negative, got {self.start_rank}"
|
||||
)
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return self.dims[0] * self.dims[1]
|
||||
|
||||
|
||||
# A tensor's placement over its side's mesh: one code per mesh axis
|
||||
# (`REPLICATE`, or the tensor dim that axis shards), or `REPLICATED` for a
|
||||
# tensor every rank holds in full.
|
||||
Placements = tuple[int, int]
|
||||
REPLICATED: Placements | None = None
|
||||
|
||||
|
||||
def check_placements(placements: Placements, context: str = "placements") -> None:
|
||||
"""Reject placement pairs m2n cannot express.
|
||||
|
||||
The header requires exactly one SHARD axis and one REPLICATE axis.
|
||||
`{REPLICATE, REPLICATE}` hits a degenerate prepare branch, which is why
|
||||
full replication is carried as `REPLICATED` and resolved separately;
|
||||
sharding both axes is not expressible at all.
|
||||
"""
|
||||
if len(placements) != MESH_NDIMS:
|
||||
raise ValueError(f"{context} must have {MESH_NDIMS} entries")
|
||||
invalid = [code for code in placements if code < REPLICATE]
|
||||
if invalid:
|
||||
raise ValueError(
|
||||
f"{context} contains invalid placement code {invalid[0]}; valid "
|
||||
f"codes are {REPLICATE} (Replicate) and non-negative tensor dimensions"
|
||||
)
|
||||
num_sharded = sum(code != REPLICATE for code in placements)
|
||||
if num_sharded == 0:
|
||||
raise ValueError(
|
||||
f"{context}: a fully replicated tensor is carried as REPLICATED, "
|
||||
"not as two "
|
||||
"REPLICATE axes"
|
||||
)
|
||||
if num_sharded == MESH_NDIMS:
|
||||
raise ValueError(
|
||||
f"{context}: nccl_m2n needs one REPLICATE mesh axis, but "
|
||||
f"{placements} shards both"
|
||||
)
|
||||
|
||||
|
||||
def resolve_layout(
|
||||
mesh: M2NMesh,
|
||||
placements: Placements | None,
|
||||
context: str = "placements",
|
||||
) -> tuple[M2NMesh, Placements]:
|
||||
"""Pair one tensor's placement with the mesh m2n should see for it.
|
||||
|
||||
A replicated tensor needs a size-1 mesh axis to carry a no-op shard, since
|
||||
m2n has no `{REPLICATE, REPLICATE}`. It is therefore described over the
|
||||
*same rank interval* re-factored as `(size, 1)`. That re-factoring is sound
|
||||
precisely because replication is order-independent — every rank holds the
|
||||
whole tensor, so it does not matter that `(a, b)` and `(size, 1)` walk the
|
||||
interval in a different order. A sharded tensor keeps its side's own
|
||||
factorization, where rank order decides who owns which shard.
|
||||
"""
|
||||
if placements is None:
|
||||
return M2NMesh((mesh.size, 1), mesh.start_rank), (REPLICATE, 0)
|
||||
check_placements(placements, context)
|
||||
return mesh, placements
|
||||
|
||||
|
||||
def shard_count(mesh: M2NMesh, placements: Placements) -> int:
|
||||
"""How many pieces this layout splits the tensor into."""
|
||||
return next(
|
||||
(mesh.dims[axis] for axis, code in enumerate(placements) if code != REPLICATE),
|
||||
1,
|
||||
)
|
||||
|
||||
|
||||
def check_plan_limits(
|
||||
src: tuple[M2NMesh, Placements],
|
||||
dst: tuple[M2NMesh, Placements],
|
||||
name: str,
|
||||
) -> None:
|
||||
"""Reject plans that exceed m2n's static per-shard fan-in / fan-out arrays.
|
||||
|
||||
Only the unambiguous cases are checked: when one side is a single shard it
|
||||
is fed by (or feeds) every shard on the other side, so the count is exactly
|
||||
the other side's shard count. In the general sharded-to-sharded case the
|
||||
overlap depends on the library's chunking, and reproducing that arithmetic
|
||||
here would duplicate internals that can change under us.
|
||||
|
||||
m2n re-checks authoritatively and, because the plan is derived from the
|
||||
shared descriptors, fails identically on every rank -- so this is about
|
||||
reporting at init with a message that names the parameter, not about
|
||||
avoiding a hang.
|
||||
"""
|
||||
src_shards = shard_count(*src)
|
||||
dst_shards = shard_count(*dst)
|
||||
if dst_shards == 1 and src_shards > MAX_SOURCE_SHARDS:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' would feed {src_shards} source shards into one "
|
||||
f"replicated destination, over m2n's MAX_SOURCES={MAX_SOURCE_SHARDS}. "
|
||||
"Shard the destination, replicate the source, or rebuild m2n with "
|
||||
"larger arrays."
|
||||
)
|
||||
if src_shards == 1 and dst_shards > MAX_DEST_SHARDS:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' would fan one source shard out to {dst_shards} "
|
||||
f"destination shards, over m2n's MAX_TARGETS={MAX_DEST_SHARDS}. "
|
||||
"Rebuild m2n with larger arrays to raise the bound."
|
||||
)
|
||||
|
||||
|
||||
def validate_layout(
|
||||
mesh: M2NMesh, placements: Placements, shape: Sequence[int], side: str
|
||||
) -> None:
|
||||
"""Check a resolved layout can describe `shape` before any collective runs.
|
||||
|
||||
Called on both sides at init so a bad plan surfaces as an error from the
|
||||
init RPC rather than as a hang inside the first reshard.
|
||||
"""
|
||||
for axis, code in enumerate(placements):
|
||||
if code == REPLICATE:
|
||||
continue
|
||||
if code >= len(shape):
|
||||
raise ValueError(
|
||||
f"{side} layout shards tensor dim {code}, but the tensor "
|
||||
f"has rank {len(shape)}"
|
||||
)
|
||||
factor = mesh.dims[axis]
|
||||
if shape[code] % factor:
|
||||
raise ValueError(
|
||||
f"{side} layout shards dim {code} (size {shape[code]}) over "
|
||||
f"{factor} ranks, which does not divide evenly"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NParamMeta(ParamMeta):
|
||||
"""`ParamMeta` extended with how the trainer places this tensor.
|
||||
|
||||
The base class carries only name / dtype / full shape, which is not enough
|
||||
to plan a reshard. `placements` is relative to its side's `M2NMesh`, or
|
||||
`REPLICATED` when every rank holds the whole tensor.
|
||||
"""
|
||||
|
||||
placements: Placements | None
|
||||
|
||||
|
||||
def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> None:
|
||||
"""Reject tensors m2n cannot move at all, with the parameter named."""
|
||||
if dtype not in SUPPORTED_DTYPES:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' has dtype {dtype}, which nccl_m2n does not "
|
||||
f"support. Supported: {sorted(str(d) for d in SUPPORTED_DTYPES)}"
|
||||
)
|
||||
if not 1 <= len(shape) <= MAX_TENSOR_DIMS:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' has rank {len(shape)}; nccl_m2n supports "
|
||||
f"rank 1..{MAX_TENSOR_DIMS}"
|
||||
)
|
||||
|
||||
|
||||
def to_mesh(m2n: Any, mesh: M2NMesh) -> Any:
|
||||
return m2n.Mesh(mesh.dims, start_rank=mesh.start_rank)
|
||||
|
||||
|
||||
def to_placements(m2n: Any, placements: Placements) -> list[Any]:
|
||||
return [
|
||||
m2n.Replicate() if code == REPLICATE else m2n.Shard(code) for code in placements
|
||||
]
|
||||
|
||||
|
||||
def publish_destination_placements(
|
||||
comm: "PyNcclCommunicator",
|
||||
first_worker_rank: int,
|
||||
placements: "Sequence[Placements | None] | None",
|
||||
num_parameters: int,
|
||||
) -> list[Placements | None]:
|
||||
"""Share the worker-side destination plan with every rank in the group.
|
||||
|
||||
The trainer must issue each reshard with the same destination the workers
|
||||
use, but once destinations are per-parameter they depend on the inference
|
||||
model, which only the workers can see. The first worker publishes them here,
|
||||
over the shared NCCL communicator; trainer ranks pass `None` and receive
|
||||
them, and the other workers pass their own so a disagreement is caught
|
||||
rather than deadlocking later.
|
||||
|
||||
Only placements travel: the destination *mesh* is still derived from the
|
||||
rank split, identically on both sides. Use the NCCL communicator directly
|
||||
because unique-id rendezvous deliberately has no bootstrap process group.
|
||||
"""
|
||||
replicated_sentinel = REPLICATE - 1
|
||||
encoded = torch.full(
|
||||
(num_parameters, MESH_NDIMS),
|
||||
replicated_sentinel,
|
||||
dtype=torch.int8,
|
||||
device=comm.device,
|
||||
)
|
||||
if comm.rank == first_worker_rank:
|
||||
if placements is None or len(placements) != num_parameters:
|
||||
raise ValueError(
|
||||
f"publishing rank needs {num_parameters} destination placements"
|
||||
)
|
||||
rows = [
|
||||
[replicated_sentinel] * MESH_NDIMS if placement is REPLICATED else placement
|
||||
for placement in placements
|
||||
]
|
||||
if rows:
|
||||
encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=comm.device))
|
||||
|
||||
comm.broadcast(encoded, src=first_worker_rank)
|
||||
return [
|
||||
REPLICATED
|
||||
if all(code == replicated_sentinel for code in row)
|
||||
else cast(Placements, tuple(row))
|
||||
for row in encoded.tolist()
|
||||
]
|
||||
|
||||
|
||||
def comm_ptr(comm: "PyNcclCommunicator") -> int:
|
||||
"""Raw `ncclComm_t` behind vLLM's `PyNcclCommunicator`.
|
||||
|
||||
m2n links its own NCCL, so the handle only means anything if vLLM loaded
|
||||
the same `libnccl.so` — set `VLLM_NCCL_SO_PATH` accordingly.
|
||||
"""
|
||||
handle = comm.comm
|
||||
ptr = getattr(handle, "value", handle)
|
||||
if not ptr:
|
||||
raise RuntimeError("PyNcclCommunicator has no live NCCL communicator")
|
||||
return int(ptr)
|
||||
@@ -0,0 +1,453 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-side weight transfer engine built on NCCL M2N (`nccl_m2n`).
|
||||
|
||||
The trainer and the inference workers share one NCCL communicator: trainer ranks
|
||||
occupy `[0, T)`, workers `[T, T + N)`. Each parameter is moved with a single
|
||||
`nccl.m2n.reshard`, which redistributes it from the trainer's layout (FSDP / EP
|
||||
/ arbitrary DTensor sharding) to the inference layout — so the trainer sends its
|
||||
local shards and never all-gathers a full tensor, which is what the broadcast
|
||||
NCCL backend forces it to do.
|
||||
|
||||
When an incoming checkpoint parameter maps directly to a live vLLM parameter,
|
||||
M2N reshards into that worker's local model storage. Parameters whose names or
|
||||
layouts cannot be resolved receive a full tensor and fall back to
|
||||
`load_weights`, preserving the existing backend's loading behavior.
|
||||
|
||||
Both meshes participate in every reshard, in the same order, so this backend has
|
||||
the same concurrency shape as the broadcast NCCL backend: the worker must be
|
||||
inside `receive_weights` while the trainer is sending. Driving that from the
|
||||
trainer is the trainer engine's job.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferEngine,
|
||||
WeightTransferInitInfo,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
DESTINATION_SHARD_AXIS,
|
||||
MESH_NDIMS,
|
||||
REPLICATED,
|
||||
M2NMesh,
|
||||
M2NParamMeta,
|
||||
Placements,
|
||||
check_plan_limits,
|
||||
check_transferable,
|
||||
comm_ptr,
|
||||
import_m2n,
|
||||
publish_destination_placements,
|
||||
resolve_layout,
|
||||
to_mesh,
|
||||
to_placements,
|
||||
validate_layout,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_layout import (
|
||||
M2NDestination,
|
||||
resolve_parameter_destinations,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
decode_nccl_unique_id,
|
||||
worker_init_process_group,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
__all__ = [
|
||||
"M2NWeightTransferInitInfo",
|
||||
"M2NWeightTransferUpdateInfo",
|
||||
"M2NWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wire types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class M2NWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Worker-side init info: the rendezvous plus the full transfer plan.
|
||||
|
||||
Layouts are static for the whole run, so they ride the one-time init
|
||||
handshake and the per-round update info stays a list of names. Everything is
|
||||
plain JSON so the HTTP control plane carries it unchanged.
|
||||
"""
|
||||
|
||||
master_address: str | None = None
|
||||
master_port: int | None = None
|
||||
rank_offset: int
|
||||
"""First worker rank, i.e. the number of trainer ranks."""
|
||||
world_size: int
|
||||
"""Trainer ranks + all inference workers."""
|
||||
src_mesh_dims: list[int]
|
||||
"""The trainer's mesh, shared by every parameter (`start_rank` is 0: the
|
||||
trainer occupies the front of the communicator)."""
|
||||
dst_mesh_dims: list[int]
|
||||
"""The inference mesh, starting at `rank_offset`. Declared rather than
|
||||
derived so both sides describe the destination identically."""
|
||||
names: list[str]
|
||||
dtype_names: list[str]
|
||||
shapes: list[list[int]]
|
||||
src_placements: list[list[int] | None]
|
||||
"""Per parameter, relative to `src_mesh_dims`; `None` means replicated."""
|
||||
nccl_unique_id_b64: str | None = field(default=None, repr=False)
|
||||
max_cta: int | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_ = self.nccl_unique_id_bytes
|
||||
num_params = len(self.names)
|
||||
for label, values in (
|
||||
("dtype_names", self.dtype_names),
|
||||
("shapes", self.shapes),
|
||||
("src_placements", self.src_placements),
|
||||
):
|
||||
if len(values) != num_params:
|
||||
raise ValueError(
|
||||
f"`{label}` should be of the same size as `names`: "
|
||||
f"got {len(values)} and {num_params}"
|
||||
)
|
||||
if self.rank_offset < 1 or self.rank_offset >= self.world_size:
|
||||
raise ValueError(
|
||||
f"`rank_offset` ({self.rank_offset}) must leave at least one "
|
||||
f"trainer rank and one worker in world_size {self.world_size}"
|
||||
)
|
||||
num_workers = self.world_size - self.rank_offset
|
||||
dst = tuple(self.dst_mesh_dims)
|
||||
if len(dst) != MESH_NDIMS or dst[0] * dst[1] != num_workers:
|
||||
raise ValueError(
|
||||
f"`dst_mesh_dims` {dst} must be {MESH_NDIMS} dims covering the "
|
||||
f"{num_workers} inference workers"
|
||||
)
|
||||
|
||||
@property
|
||||
def nccl_unique_id_bytes(self) -> bytes | None:
|
||||
return decode_nccl_unique_id(
|
||||
master_address=self.master_address,
|
||||
master_port=self.master_port,
|
||||
nccl_unique_id_b64=self.nccl_unique_id_b64,
|
||||
ctx="M2NWeightTransferInitInfo",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class M2NWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Per-round update info: which parameters this chunk carries, in order.
|
||||
|
||||
Shapes, dtypes and layouts were fixed at init, so a round only needs to say
|
||||
what is coming and in what order — both sides must issue their reshards in
|
||||
exactly that order.
|
||||
"""
|
||||
|
||||
names: list[str]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Worker engine
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class M2NWeightTransferEngine(
|
||||
WeightTransferEngine[M2NWeightTransferInitInfo, M2NWeightTransferUpdateInfo]
|
||||
):
|
||||
"""Inference-side engine: receives each parameter with one reshard.
|
||||
|
||||
Resolvable parameters are resharded from the trainer layout directly into
|
||||
each worker's live local model storage. Unresolvable parameters receive a
|
||||
full tensor and use `load_weights`. In both cases, the trainer sends its
|
||||
local shards without materializing a full tensor.
|
||||
"""
|
||||
|
||||
init_info_cls = M2NWeightTransferInitInfo
|
||||
update_info_cls = M2NWeightTransferUpdateInfo
|
||||
# The destination plan is resolved against the model this engine was built
|
||||
# for, so it cannot be retargeted at a draft model mid-flight.
|
||||
supports_draft_weight_update = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
self._m2n: Any = None
|
||||
self._handle: Any = None
|
||||
self._metas: list[M2NParamMeta] = []
|
||||
self._src_mesh: M2NMesh | None = None
|
||||
self._dst_mesh: M2NMesh | None = None
|
||||
self._parameter_destinations: list[M2NDestination] = []
|
||||
self._index: dict[str, int] = {}
|
||||
self._uses_load_weights = False
|
||||
|
||||
def init_transfer_engine(self, init_info: M2NWeightTransferInitInfo) -> None:
|
||||
"""Join the trainer's communicator and rebuild the transfer plan.
|
||||
|
||||
Every precondition (dtype, tensor rank, divisibility) is checked here so
|
||||
a bad plan fails the init RPC instead of hanging the first collective.
|
||||
"""
|
||||
self._m2n = import_m2n()
|
||||
self._metas = []
|
||||
|
||||
self._src_mesh = M2NMesh(
|
||||
cast(tuple[int, int], tuple(init_info.src_mesh_dims)), 0
|
||||
)
|
||||
if self._src_mesh.size != init_info.rank_offset:
|
||||
raise ValueError(
|
||||
f"source mesh covers {self._src_mesh.size} ranks, but there are "
|
||||
f"{init_info.rank_offset} trainer ranks"
|
||||
)
|
||||
# The inference topology, as declared by the trainer. This is the mesh
|
||||
# a sharded destination is placed over; a replicated one is described
|
||||
# by the alternate descriptor `resolve_layout` derives from it, which
|
||||
# covers the same ranks but is not this mesh.
|
||||
self._dst_mesh = M2NMesh(
|
||||
cast(tuple[int, int], tuple(init_info.dst_mesh_dims)),
|
||||
init_info.rank_offset,
|
||||
)
|
||||
|
||||
for name, name_dtype, shape, placements in zip(
|
||||
init_info.names,
|
||||
init_info.dtype_names,
|
||||
init_info.shapes,
|
||||
init_info.src_placements,
|
||||
):
|
||||
dtype = getattr(torch, name_dtype, None)
|
||||
if not isinstance(dtype, torch.dtype):
|
||||
raise ValueError(
|
||||
f"parameter '{name}' has invalid dtype name "
|
||||
f"'{name_dtype}'; expected the name of a torch.dtype"
|
||||
)
|
||||
check_transferable(name, dtype, shape)
|
||||
codes = (
|
||||
REPLICATED
|
||||
if placements is None
|
||||
else cast(Placements, tuple(placements))
|
||||
)
|
||||
src_mesh, src_placements = resolve_layout(
|
||||
self._src_mesh, codes, f"parameter '{name}' source placements"
|
||||
)
|
||||
validate_layout(src_mesh, src_placements, shape, "source")
|
||||
# Destination checks are deferred until loader-aware resolution;
|
||||
# assuming replication here can reject a valid sharded plan.
|
||||
self._metas.append(M2NParamMeta(name, dtype, tuple(shape), codes))
|
||||
|
||||
parallel_config = self.parallel_config
|
||||
shard_axis_size = self._dst_mesh.dims[DESTINATION_SHARD_AXIS]
|
||||
quantization = getattr(self.model_config, "quantization", None)
|
||||
# A quantized weight loader may dequantize / transpose / requantize, and
|
||||
# under PP a rank's local shape is not the checkpoint shape split over
|
||||
# the shard axis. Both break shape-derived resolution, so take the
|
||||
# fallback.
|
||||
allow_direct = (
|
||||
quantization is None and parallel_config.pipeline_parallel_size == 1
|
||||
)
|
||||
# Match every incoming checkpoint parameter to its rank-local target.
|
||||
# Resolvable parameters can be resharded directly into live model
|
||||
# storage; the rest receive a full tensor for `load_weights`.
|
||||
self._parameter_destinations = resolve_parameter_destinations(
|
||||
self.model,
|
||||
[m.name for m in self._metas],
|
||||
[m.dtype for m in self._metas],
|
||||
[m.shape for m in self._metas],
|
||||
num_workers=self._dst_mesh.size,
|
||||
shard_axis_size=shard_axis_size,
|
||||
allow_direct=allow_direct,
|
||||
)
|
||||
# Check the resolved layout rather than a provisional replicated one.
|
||||
# For example, 32 source shards feeding 4 destination shards may need
|
||||
# only 8 sources per destination, while a replicated plan needs all 32.
|
||||
for meta, destination in zip(self._metas, self._parameter_destinations):
|
||||
src_mesh, src_placements = resolve_layout(
|
||||
self._src_mesh,
|
||||
meta.placements,
|
||||
f"parameter '{meta.name}' source placements",
|
||||
)
|
||||
dst_mesh, dst_placements = resolve_layout(
|
||||
self._dst_mesh,
|
||||
destination.placements,
|
||||
f"parameter '{meta.name}' destination placements",
|
||||
)
|
||||
validate_layout(dst_mesh, dst_placements, meta.shape, "destination")
|
||||
check_plan_limits(
|
||||
(src_mesh, src_placements),
|
||||
(dst_mesh, dst_placements),
|
||||
meta.name,
|
||||
)
|
||||
|
||||
# Update requests carry names only, so cache their plan indices. Track
|
||||
# whether any fallback entry requires the `load_weights` lifecycle.
|
||||
self._index = {meta.name: i for i, meta in enumerate(self._metas)}
|
||||
self._uses_load_weights = any(
|
||||
not destination.direct for destination in self._parameter_destinations
|
||||
)
|
||||
|
||||
self.model_update_group = worker_init_process_group(
|
||||
NCCLWeightTransferInitInfo(
|
||||
master_address=init_info.master_address,
|
||||
master_port=init_info.master_port,
|
||||
nccl_unique_id_b64=init_info.nccl_unique_id_b64,
|
||||
rank_offset=init_info.rank_offset,
|
||||
world_size=init_info.world_size,
|
||||
),
|
||||
parallel_config,
|
||||
)
|
||||
# Destinations are per-parameter and depend on the inference model, so
|
||||
# the trainer cannot derive them. The first worker publishes the plan
|
||||
# and every other rank checks its own against it — a silent
|
||||
# disagreement between workers would mean mismatched collectives.
|
||||
mine = [destination.placements for destination in self._parameter_destinations]
|
||||
agreed = publish_destination_placements(
|
||||
self.model_update_group, init_info.rank_offset, mine, len(self._metas)
|
||||
)
|
||||
if agreed != mine:
|
||||
mismatched = next(
|
||||
meta.name for meta, a, b in zip(self._metas, mine, agreed) if a != b
|
||||
)
|
||||
raise RuntimeError(
|
||||
"inference workers resolved different nccl_m2n destinations "
|
||||
f"(first mismatch: '{mismatched}'); all workers must run the "
|
||||
"same model and parallel config"
|
||||
)
|
||||
|
||||
self._handle = self._m2n.Handle.create(
|
||||
self._m2n.Config(max_cta=init_info.max_cta)
|
||||
)
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Set up layerwise reloading, but only if some parameter needs it.
|
||||
|
||||
Directly-resharded parameters are written in place and never go through
|
||||
`load_weights`, so a plan with no fallback entries has nothing to
|
||||
reload.
|
||||
"""
|
||||
if not self._uses_load_weights:
|
||||
return
|
||||
from vllm.model_executor.model_loader.reload import initialize_layerwise_reload
|
||||
|
||||
initialize_layerwise_reload(self.model)
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Finalize layerwise reloading when the plan uses fallback entries."""
|
||||
if not self._uses_load_weights:
|
||||
return
|
||||
from vllm.model_executor.model_loader.reload import finalize_layerwise_reload
|
||||
|
||||
finalize_layerwise_reload(self.model, self.model_config)
|
||||
|
||||
def receive_weights(self, update_info: M2NWeightTransferUpdateInfo) -> None:
|
||||
"""Receive each requested parameter using its initialization-time plan."""
|
||||
if self._handle is None or self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
|
||||
requested = []
|
||||
for name in update_info.names:
|
||||
index = self._index.get(name)
|
||||
if index is None:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' was not declared at init; the "
|
||||
"trainer must send the same parameter set it announced"
|
||||
)
|
||||
requested.append((name, index))
|
||||
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
disable_mtp_completeness_check,
|
||||
)
|
||||
|
||||
comm = comm_ptr(self.model_update_group)
|
||||
stream = torch.cuda.current_stream()
|
||||
|
||||
# Reshard lazily so one `load_weights` invocation sees the complete
|
||||
# fallback sequence without staging every full tensor at once. Direct
|
||||
# destinations still execute in request order as the iterator advances.
|
||||
def reshard_requested_weights() -> Iterator[tuple[str, torch.Tensor]]:
|
||||
for name, index in requested:
|
||||
meta = self._metas[index]
|
||||
destination = self._parameter_destinations[index]
|
||||
buffer = destination.tensor
|
||||
if buffer is None:
|
||||
buffer = torch.empty(
|
||||
meta.shape, dtype=meta.dtype, device=self.device
|
||||
)
|
||||
|
||||
self._reshard(comm, stream, meta, destination.placements, buffer)
|
||||
|
||||
if destination.tensor is None:
|
||||
# `load_weights` reads on the host stream, so the transfer
|
||||
# has to have landed before it runs.
|
||||
stream.synchronize()
|
||||
yield name, buffer
|
||||
|
||||
has_fallback = any(
|
||||
self._parameter_destinations[index].tensor is None for _, index in requested
|
||||
)
|
||||
with disable_mtp_completeness_check():
|
||||
received_weights = reshard_requested_weights()
|
||||
if has_fallback:
|
||||
# Some model loaders keep invocation-local state to combine
|
||||
# related checkpoint tensors, so preserve one iterable scope.
|
||||
self.model.load_weights(received_weights)
|
||||
else:
|
||||
# The iterator performs the reshards. With nothing to yield to
|
||||
# `load_weights`, exhaust it here to execute direct transfers.
|
||||
for _ in received_weights:
|
||||
pass
|
||||
|
||||
def _reshard(
|
||||
self,
|
||||
comm: int,
|
||||
stream: torch.cuda.Stream,
|
||||
src_meta: M2NParamMeta,
|
||||
dst_placements: Placements | None,
|
||||
dst_buffer: torch.Tensor,
|
||||
) -> None:
|
||||
"""Reshard one parameter from its trainer layout into a worker buffer.
|
||||
|
||||
`dst_placements` describes the buffer's logical placement over the
|
||||
worker mesh; `dst_buffer` is this rank's physical storage.
|
||||
"""
|
||||
assert self._src_mesh is not None
|
||||
assert self._dst_mesh is not None
|
||||
m2n = self._m2n
|
||||
src_mesh, resolved_src_placements = resolve_layout(
|
||||
self._src_mesh, src_meta.placements
|
||||
)
|
||||
dst_mesh, resolved_dst_placements = resolve_layout(
|
||||
self._dst_mesh, dst_placements
|
||||
)
|
||||
m2n.reshard(
|
||||
None,
|
||||
dst_buffer,
|
||||
comm,
|
||||
stream,
|
||||
src_mesh=to_mesh(m2n, src_mesh),
|
||||
src_placements=to_placements(m2n, resolved_src_placements),
|
||||
dst_mesh=to_mesh(m2n, dst_mesh),
|
||||
dst_placements=to_placements(m2n, resolved_dst_placements),
|
||||
handle=self._handle,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Finish pending GPU work and release M2N communication state."""
|
||||
if self._handle is not None:
|
||||
# M2N does not synchronize caller streams on finalize.
|
||||
torch.accelerator.synchronize()
|
||||
self._handle.destroy()
|
||||
self._handle = None
|
||||
self.model_update_group = None
|
||||
self._parameter_destinations = []
|
||||
@@ -0,0 +1,261 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Destination-layout resolution for the NCCL M2N weight transfer backend.
|
||||
|
||||
For every incoming checkpoint parameter the worker needs a destination buffer
|
||||
plus its placement over the inference mesh. Two outcomes are possible:
|
||||
|
||||
* **direct** — when the model is not quantized, the parameter maps 1:1 onto a
|
||||
live vLLM parameter and its loader is a small, known-safe copy or tensor-
|
||||
parallel loader. The loader's declared input/output dimension determines the
|
||||
placement; shapes are validation, not an inference mechanism. The reshard
|
||||
writes straight into the live parameter, so each rank receives only its own
|
||||
shard and nothing is copied afterwards.
|
||||
* **fallback** — anything else. The reshard delivers the whole tensor to every
|
||||
rank and `load_weights` does the sharding, exactly as the broadcast NCCL
|
||||
backend does. Fused parameters (`qkv_proj`, `gate_up_proj`, MoE `w13`/`w2`)
|
||||
take this path: the checkpoint name does not name a vLLM parameter, so there
|
||||
is nothing to resolve against.
|
||||
|
||||
Correctness never depends on a parameter resolving — the fallback is always
|
||||
available and is the same path the existing backend uses.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
DESTINATION_REPLICA_AXIS,
|
||||
DESTINATION_SHARD_AXIS,
|
||||
MESH_NDIMS,
|
||||
REPLICATE,
|
||||
REPLICATED,
|
||||
Placements,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_DIRECT_WEIGHT_LOADING_BLOCKLIST = {
|
||||
"vllm.model_executor.models.gpt2.GPT2Model",
|
||||
}
|
||||
|
||||
|
||||
def _module_type_name(module: torch.nn.Module) -> str:
|
||||
cls = type(module)
|
||||
return f"{cls.__module__}.{cls.__name__}"
|
||||
|
||||
|
||||
def _destination_placements(shard_dim: int) -> Placements:
|
||||
"""Place one tensor shard on the configured destination mesh axis."""
|
||||
placements = [REPLICATE] * MESH_NDIMS
|
||||
placements[DESTINATION_REPLICA_AXIS] = REPLICATE
|
||||
placements[DESTINATION_SHARD_AXIS] = shard_dim
|
||||
return cast(Placements, tuple(placements))
|
||||
|
||||
|
||||
@dataclass
|
||||
class M2NDestination:
|
||||
"""Where one checkpoint parameter lands on this worker."""
|
||||
|
||||
name: str
|
||||
placements: Placements | None
|
||||
"""Placement over the inference mesh, or `REPLICATED` for the fallback."""
|
||||
tensor: torch.Tensor | None
|
||||
"""The live parameter view to reshard into, or None for the fallback path
|
||||
(the engine allocates a full replica per round and calls `load_weights`)."""
|
||||
|
||||
@property
|
||||
def direct(self) -> bool:
|
||||
return self.tensor is not None
|
||||
|
||||
|
||||
def _base_loader_shard_dim(
|
||||
param: torch.nn.Parameter,
|
||||
shard_axis_size: int,
|
||||
) -> int | None:
|
||||
"""Return a known-safe loader's declared shard dim, or reject it.
|
||||
|
||||
Loader identity is deliberately fail-closed. A missing loader is not
|
||||
treated as ``default_weight_loader`` because a model-level ``load_weights``
|
||||
implementation may transform the tensor before it reaches that default.
|
||||
Wrappers and partials are also rejected: recognizing their wrapped callable
|
||||
would ignore behavior added by the wrapper.
|
||||
"""
|
||||
loader = getattr(param, "weight_loader", None)
|
||||
if loader is None or isinstance(loader, partial):
|
||||
return None
|
||||
|
||||
owner = loader.__self__ if inspect.ismethod(loader) else None
|
||||
loader_fn = loader.__func__ if inspect.ismethod(loader) else loader
|
||||
|
||||
# Lazy imports avoid making the weight-transfer registry import all model
|
||||
# executor layers during normal vLLM startup.
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
replicated_loaders = {
|
||||
default_weight_loader,
|
||||
ReplicatedLinear.weight_loader,
|
||||
}
|
||||
column_loaders = {
|
||||
ColumnParallelLinear.weight_loader,
|
||||
ColumnParallelLinear.weight_loader_v2,
|
||||
}
|
||||
row_loaders = {
|
||||
RowParallelLinear.weight_loader,
|
||||
RowParallelLinear.weight_loader_v2,
|
||||
}
|
||||
|
||||
# These attributes signal packing, a transform, or pre-sharded checkpoint
|
||||
# storage. Such semantics cannot be reproduced by a plain M2N placement.
|
||||
# Check them before accepting replicated loaders too: model-level loading
|
||||
# may transform a tensor before handing it to an otherwise plain copier.
|
||||
metadata_attrs = ("packed_dim", "packed_factor", "marlin_tile_size")
|
||||
flag_attrs = ("needs_scalar_to_array", "is_sharded_weight", "is_transposed")
|
||||
if any(getattr(param, attr, None) is not None for attr in metadata_attrs):
|
||||
return None
|
||||
if any(bool(getattr(param, attr, False)) for attr in flag_attrs):
|
||||
return None
|
||||
|
||||
if loader_fn in replicated_loaders:
|
||||
return REPLICATE
|
||||
if loader_fn not in column_loaders | row_loaders:
|
||||
return None
|
||||
|
||||
if owner is None or getattr(owner, "tp_size", None) != shard_axis_size:
|
||||
return None
|
||||
|
||||
attr = "output_dim" if loader_fn in column_loaders else "input_dim"
|
||||
dim = getattr(param, attr, None)
|
||||
return dim if isinstance(dim, int) and dim >= 0 else None
|
||||
|
||||
|
||||
def _validated_shard_dim(
|
||||
global_shape: Sequence[int],
|
||||
local_shape: Sequence[int],
|
||||
declared_dim: int,
|
||||
shard_axis_size: int,
|
||||
) -> int | None:
|
||||
"""Validate a loader-declared shard dimension against both shapes."""
|
||||
if len(global_shape) != len(local_shape):
|
||||
return None
|
||||
|
||||
if declared_dim == REPLICATE:
|
||||
return REPLICATE if tuple(global_shape) == tuple(local_shape) else None
|
||||
if declared_dim >= len(global_shape):
|
||||
return None
|
||||
for dim, (whole, local) in enumerate(zip(global_shape, local_shape)):
|
||||
expected = local * shard_axis_size if dim == declared_dim else local
|
||||
if whole != expected:
|
||||
return None
|
||||
return declared_dim
|
||||
|
||||
|
||||
def resolve_parameter_destinations(
|
||||
model: torch.nn.Module,
|
||||
names: Sequence[str],
|
||||
dtypes: Sequence[torch.dtype],
|
||||
shapes: Sequence[Sequence[int]],
|
||||
*,
|
||||
num_workers: int,
|
||||
shard_axis_size: int,
|
||||
allow_direct: bool,
|
||||
) -> list[M2NDestination]:
|
||||
"""Build this worker's destination plan, one entry per checkpoint parameter.
|
||||
|
||||
Placements are relative to the inference mesh: axis 0 replicates and axis 1
|
||||
shards. `allow_direct=False` forces every parameter onto the fallback path;
|
||||
the engine sets it for pipeline-parallel or quantized deployments, where a
|
||||
parameter's local shape is not simply the checkpoint shape split across the
|
||||
shard axis. Parameters below a model type with known model-level checkpoint
|
||||
preprocessing also fall back, since bypassing `load_weights` would skip the
|
||||
transformation.
|
||||
"""
|
||||
params = dict(model.named_parameters()) if allow_direct else {}
|
||||
blocked_param_ids = {
|
||||
id(param)
|
||||
for module in model.modules()
|
||||
if _module_type_name(module) in _DIRECT_WEIGHT_LOADING_BLOCKLIST
|
||||
for param in module.parameters()
|
||||
}
|
||||
|
||||
destinations: list[M2NDestination] = []
|
||||
for name, dtype, shape in zip(names, dtypes, shapes):
|
||||
param = params.get(name)
|
||||
dim = None
|
||||
if (
|
||||
param is not None
|
||||
and id(param) not in blocked_param_ids
|
||||
and param.dtype == dtype
|
||||
and param.data.is_contiguous()
|
||||
and num_workers % shard_axis_size == 0
|
||||
):
|
||||
declared_dim = _base_loader_shard_dim(param, shard_axis_size)
|
||||
if declared_dim is not None:
|
||||
dim = _validated_shard_dim(
|
||||
shape, param.shape, declared_dim, shard_axis_size
|
||||
)
|
||||
|
||||
if dim is None:
|
||||
# A replicated parameter is identical on every rank, so it needs no
|
||||
# placement of its own — REPLICATED lets resolve_layout spread it
|
||||
# over all the workers regardless of how the mesh is factored.
|
||||
destinations.append(M2NDestination(name, REPLICATED, None))
|
||||
elif dim == REPLICATE:
|
||||
assert param is not None
|
||||
destinations.append(M2NDestination(name, REPLICATED, param.data))
|
||||
else:
|
||||
assert param is not None
|
||||
destinations.append(
|
||||
M2NDestination(name, _destination_placements(dim), param.data)
|
||||
)
|
||||
|
||||
# Layerwise reload finalizes a module as a unit. Mixing direct and fallback
|
||||
# parameters within one module can copy stale storage over the direct load.
|
||||
fallback_modules = {
|
||||
name.rpartition(".")[0]
|
||||
for name, destination in zip(names, destinations)
|
||||
if not destination.direct and name in params
|
||||
}
|
||||
destinations = [
|
||||
M2NDestination(destination.name, REPLICATED, None)
|
||||
if destination.direct
|
||||
and destination.name.rpartition(".")[0] in fallback_modules
|
||||
else destination
|
||||
for destination in destinations
|
||||
]
|
||||
|
||||
num_direct = sum(d.direct for d in destinations)
|
||||
parameter_bytes = [
|
||||
math.prod(shape) * dtype.itemsize for dtype, shape in zip(dtypes, shapes)
|
||||
]
|
||||
total_bytes = sum(parameter_bytes)
|
||||
direct_bytes = sum(
|
||||
nbytes
|
||||
for destination, nbytes in zip(destinations, parameter_bytes)
|
||||
if destination.direct
|
||||
)
|
||||
direct_byte_percentage = 100 * direct_bytes / total_bytes if total_bytes else 0
|
||||
logger.info(
|
||||
"nccl_m2n destination plan: %d/%d parameters resharded directly into "
|
||||
"the model, %d via full-tensor fallback; direct byte coverage: "
|
||||
"%d/%d bytes (%.1f%%)",
|
||||
num_direct,
|
||||
len(destinations),
|
||||
len(destinations) - num_direct,
|
||||
direct_bytes,
|
||||
total_bytes,
|
||||
direct_byte_percentage,
|
||||
)
|
||||
return destinations
|
||||
@@ -0,0 +1,155 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Trainer-side weight sources for the NCCL M2N backend.
|
||||
|
||||
m2n plans a transfer from both sides' layouts, so the trainer has to say how it
|
||||
holds each parameter — which the base `WeightSource` / `ParamMeta` pair does not
|
||||
express. This module supplies the m2n flavor of both, and the DTensor-backed
|
||||
implementation that covers the common trainer.
|
||||
|
||||
A source declares its mesh once (`mesh()`) and a placement per parameter, since
|
||||
one topology describes every tensor on the side.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.base import ParamMeta, WeightSource
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
REPLICATE,
|
||||
REPLICATED,
|
||||
M2NMesh,
|
||||
M2NParamMeta,
|
||||
Placements,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"M2NWeightSource",
|
||||
"DTensorModuleSource",
|
||||
"mesh_from_tensor",
|
||||
"placements_from_tensor",
|
||||
]
|
||||
|
||||
|
||||
class M2NWeightSource(WeightSource):
|
||||
"""A `WeightSource` that also describes how the trainer holds its weights.
|
||||
|
||||
Unlike `ModuleSource`, iteration yields each rank's **local shard**, not a
|
||||
materialized full tensor: gathering is exactly the cost m2n removes.
|
||||
"""
|
||||
|
||||
def mesh(self) -> M2NMesh:
|
||||
"""The trainer's rank topology, shared by every parameter."""
|
||||
raise NotImplementedError
|
||||
|
||||
def metadata(self) -> list[ParamMeta]:
|
||||
"""Name, dtype, full shape, and trainer placement for each parameter."""
|
||||
raise NotImplementedError
|
||||
|
||||
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""Yield `(name, local shard)` pairs in the same order as `metadata()`."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _placement_code(placement: Any) -> int:
|
||||
"""Map a `torch.distributed` placement onto an m2n placement code."""
|
||||
name = type(placement).__name__
|
||||
if name == "Replicate":
|
||||
return REPLICATE
|
||||
if name == "Shard":
|
||||
return int(placement.dim)
|
||||
raise ValueError(
|
||||
f"nccl_m2n cannot express the {name} placement; only Replicate and "
|
||||
"Shard are supported"
|
||||
)
|
||||
|
||||
|
||||
def mesh_from_tensor(tensor: torch.Tensor, num_trainer_ranks: int) -> M2NMesh:
|
||||
"""The mesh a parameter lives on, as an `M2NMesh`.
|
||||
|
||||
A plain tensor is identical on every trainer rank, so it spans all of them.
|
||||
"""
|
||||
device_mesh = getattr(tensor, "device_mesh", None)
|
||||
if device_mesh is None:
|
||||
return M2NMesh((num_trainer_ranks, 1), 0)
|
||||
|
||||
grid = device_mesh.mesh
|
||||
ranks = grid.flatten().tolist()
|
||||
if ranks != list(range(len(ranks))):
|
||||
raise ValueError(
|
||||
"nccl_m2n requires the trainer's device mesh to cover the "
|
||||
f"contiguous rank interval [0, {len(ranks)}); got {ranks}"
|
||||
)
|
||||
if grid.ndim > 2:
|
||||
raise ValueError(
|
||||
f"nccl_m2n supports 1-D and 2-D device meshes, got {grid.ndim}-D"
|
||||
)
|
||||
dims = tuple(grid.shape)
|
||||
return M2NMesh(dims if len(dims) == 2 else (dims[0], 1), 0)
|
||||
|
||||
|
||||
def placements_from_tensor(tensor: torch.Tensor) -> Placements | None:
|
||||
"""How a parameter is placed over its mesh, or `REPLICATED`.
|
||||
|
||||
`REPLICATED` is returned rather than a `(REPLICATE, REPLICATE)` pair: m2n
|
||||
cannot express that, and the size-1-axis encoding it needs instead is
|
||||
applied later by `resolve_layout`, which owns that workaround.
|
||||
"""
|
||||
placements = getattr(tensor, "placements", None)
|
||||
if placements is None:
|
||||
return REPLICATED
|
||||
codes = [_placement_code(p) for p in placements]
|
||||
if all(code == REPLICATE for code in codes):
|
||||
return REPLICATED
|
||||
if len(codes) == 1:
|
||||
return (codes[0], REPLICATE)
|
||||
return (codes[0], codes[1])
|
||||
|
||||
|
||||
class DTensorModuleSource(M2NWeightSource):
|
||||
"""`M2NWeightSource` over `module.named_parameters()`.
|
||||
|
||||
Covers both the FSDP/DTensor trainer (placement read off each parameter,
|
||||
local shard yielded via `to_local()`) and the plain replicated trainer, with
|
||||
no special casing. Trainers with a custom producer (a Megatron export, MoE
|
||||
re-fusing) subclass `M2NWeightSource` instead.
|
||||
"""
|
||||
|
||||
def __init__(self, module: torch.nn.Module, num_trainer_ranks: int) -> None:
|
||||
"""Wrap `module`; `num_trainer_ranks` sizes the mesh for plain tensors."""
|
||||
self._module = module
|
||||
self._num_trainer_ranks = num_trainer_ranks
|
||||
|
||||
def mesh(self) -> M2NMesh:
|
||||
"""The mesh every sharded parameter agrees on.
|
||||
|
||||
Replicated parameters span the trainer without a device mesh of their
|
||||
own, so they never decide this; a model whose *sharded* parameters
|
||||
disagree cannot be described by one side mesh and is rejected.
|
||||
"""
|
||||
meshes = {
|
||||
mesh_from_tensor(p, self._num_trainer_ranks)
|
||||
for _, p in self._module.named_parameters()
|
||||
if placements_from_tensor(p) is not REPLICATED
|
||||
}
|
||||
if len(meshes) > 1:
|
||||
raise ValueError(
|
||||
"nccl_m2n needs one mesh per side, but this module's sharded "
|
||||
f"parameters span several: {sorted(m.dims for m in meshes)}"
|
||||
)
|
||||
return meshes.pop() if meshes else M2NMesh((self._num_trainer_ranks, 1), 0)
|
||||
|
||||
def metadata(self) -> list[ParamMeta]:
|
||||
"""Read global shape/dtype and placements without gathering shards."""
|
||||
return [
|
||||
M2NParamMeta(name, p.dtype, tuple(p.shape), placements_from_tensor(p))
|
||||
for name, p in self._module.named_parameters()
|
||||
]
|
||||
|
||||
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""Yield each parameter's local shard (`to_local()`), or the tensor itself."""
|
||||
for name, param in self._module.named_parameters():
|
||||
to_local = getattr(param, "to_local", None)
|
||||
yield name, (to_local() if callable(to_local) else param)
|
||||
@@ -0,0 +1,414 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Trainer-side weight transfer engine for the NCCL M2N backend.
|
||||
|
||||
Symmetric to `M2NWeightTransferEngine` but in the training process. Every
|
||||
trainer rank joins the shared communicator and runs every reshard, sending its
|
||||
own local shard; only rank 0 touches the inference control plane.
|
||||
|
||||
Workers join the communicator during init and run reshard from inside
|
||||
`update_weights`, so those two RPCs overlap with work on this side. Both run on
|
||||
a helper thread and are joined afterwards, which keeps that overlap inside the
|
||||
engine -- where the `TrainerWeightTransferEngine` contract puts it -- rather than
|
||||
in every caller.
|
||||
"""
|
||||
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, cast
|
||||
|
||||
import torch
|
||||
from typing_extensions import Self
|
||||
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
TrainerInitInfo,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
M2NMesh,
|
||||
M2NParamMeta,
|
||||
Placements,
|
||||
check_transferable,
|
||||
comm_ptr,
|
||||
import_m2n,
|
||||
publish_destination_placements,
|
||||
resolve_layout,
|
||||
to_mesh,
|
||||
to_placements,
|
||||
validate_layout,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_source import M2NWeightSource
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
decode_nccl_unique_id,
|
||||
trainer_init,
|
||||
uid_init_process_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = ["M2NTrainerInitInfo", "M2NTrainerWeightTransferEngine"]
|
||||
|
||||
|
||||
def _dtype_name(dtype: torch.dtype) -> str:
|
||||
"""Short name for a dtype, e.g. `bfloat16`, for the worker-side init dict."""
|
||||
return str(dtype).split(".")[-1]
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class M2NTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for nccl_m2n.
|
||||
|
||||
`rank` (from `TrainerInitInfo`) is this trainer process's rank; it is also
|
||||
its rank in the shared communicator, since the trainer occupies
|
||||
`[0, num_trainer_ranks)`. Rank 0 drives the control plane.
|
||||
"""
|
||||
|
||||
backend: ClassVar[str] = "nccl_m2n"
|
||||
|
||||
master_address: str | None = None
|
||||
master_port: int | None = None
|
||||
world_size: int
|
||||
"""Trainer ranks + all inference workers."""
|
||||
num_trainer_ranks: int = 1
|
||||
dst_mesh_dims: tuple[int, int] | None = None
|
||||
"""How the inference ranks are laid out: axis 0 replicates and axis 1
|
||||
shards. The trainer declares it so both sides describe the destination
|
||||
identically; defaults to a flat `(num_workers, 1)`, which is all a
|
||||
replicated destination needs."""
|
||||
max_cta: int | None = None
|
||||
nccl_unique_id_b64: str | None = field(default=None, repr=False)
|
||||
|
||||
@property
|
||||
def destination_mesh_dims(self) -> tuple[int, int]:
|
||||
"""`dst_mesh_dims`, or a flat mesh over every inference worker."""
|
||||
if self.dst_mesh_dims is not None:
|
||||
return self.dst_mesh_dims
|
||||
return (self.world_size - self.num_trainer_ranks, 1)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Reject rank counts or destination meshes that cannot form a group."""
|
||||
_ = self.nccl_unique_id_bytes
|
||||
if not 0 < self.num_trainer_ranks < self.world_size:
|
||||
raise ValueError(
|
||||
f"`num_trainer_ranks` ({self.num_trainer_ranks}) must leave at "
|
||||
f"least one worker in world_size {self.world_size}"
|
||||
)
|
||||
num_workers = self.world_size - self.num_trainer_ranks
|
||||
if self.dst_mesh_dims is not None:
|
||||
dims = tuple(self.dst_mesh_dims)
|
||||
if len(dims) != 2 or dims[0] * dims[1] != num_workers:
|
||||
raise ValueError(
|
||||
f"`dst_mesh_dims` {dims} must be two dims covering the "
|
||||
f"{num_workers} inference workers"
|
||||
)
|
||||
if not 0 <= self.rank < self.num_trainer_ranks:
|
||||
raise ValueError(
|
||||
f"trainer `rank` ({self.rank}) must be below "
|
||||
f"`num_trainer_ranks` ({self.num_trainer_ranks})"
|
||||
)
|
||||
|
||||
@property
|
||||
def nccl_unique_id_bytes(self) -> bytes | None:
|
||||
return decode_nccl_unique_id(
|
||||
master_address=self.master_address,
|
||||
master_port=self.master_port,
|
||||
nccl_unique_id_b64=self.nccl_unique_id_b64,
|
||||
ctx="M2NTrainerInitInfo",
|
||||
)
|
||||
|
||||
|
||||
class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitInfo]):
|
||||
"""Trainer-side engine: sends every rank's local shard, once per parameter.
|
||||
|
||||
Called on every trainer rank. All ranks join the communicator and run every
|
||||
reshard; only rank 0 touches the control plane.
|
||||
"""
|
||||
|
||||
init_info_cls = M2NTrainerInitInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: M2NWeightSource,
|
||||
is_controller: bool,
|
||||
num_trainer_ranks: int,
|
||||
) -> None:
|
||||
"""Hold the client and source; `trainer_init` fills in the group."""
|
||||
super().__init__(client=client, source=source, is_sender=is_controller)
|
||||
self.source: M2NWeightSource = source
|
||||
self.is_controller = is_controller
|
||||
self.num_trainer_ranks = num_trainer_ranks
|
||||
self.group: PyNcclCommunicator | None = None
|
||||
self._m2n: Any = None
|
||||
self._handle: Any = None
|
||||
self._metas: list[M2NParamMeta] = []
|
||||
self._src_mesh: M2NMesh | None = None
|
||||
self._dst_mesh: M2NMesh | None = None
|
||||
self._dst_placements: list[Placements | None] = []
|
||||
self._executor: ThreadPoolExecutor | None = None
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
init_info: M2NTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
"""Build the engine and rendezvous with the inference side.
|
||||
|
||||
Runs on *every* trainer rank. Rank 0 additionally ships the transfer
|
||||
plan to the workers and drives the control plane; the other ranks only
|
||||
build local state and join the communicator. Every rank participates
|
||||
in every reshard and sends its local shard.
|
||||
|
||||
The ordering here is not arbitrary -- see the comments inline. In
|
||||
short: validate before anything blocks; start the init RPC without
|
||||
waiting (workers join the communicator inside that RPC, so it cannot
|
||||
return until this side has joined too); then join the NCCL
|
||||
communicator; then create the M2N handle; and only then wait for the RPC
|
||||
to finish.
|
||||
"""
|
||||
# The base WeightSource carries name/dtype/shape but not layout, which
|
||||
# is not enough to plan a reshard. Fail here rather than at the first
|
||||
# missing attribute deep inside the send loop.
|
||||
if not isinstance(source, M2NWeightSource):
|
||||
raise TypeError(
|
||||
"nccl_m2n needs per-parameter layouts, so its source must be an "
|
||||
f"M2NWeightSource (e.g. DTensorModuleSource); got "
|
||||
f"{type(source).__name__}"
|
||||
)
|
||||
|
||||
# Trainer-side weight transfer engine for this rank. `__init__` only
|
||||
# holds the client and source; the communicator, meshes, and M2N handle
|
||||
# are filled in below.
|
||||
engine = cls(
|
||||
client=client,
|
||||
source=source,
|
||||
is_controller=init_info.rank == 0,
|
||||
num_trainer_ranks=init_info.num_trainer_ranks,
|
||||
)
|
||||
engine._m2n = import_m2n()
|
||||
|
||||
engine._src_mesh = source.mesh()
|
||||
if engine._src_mesh.size != init_info.num_trainer_ranks:
|
||||
raise ValueError(
|
||||
f"source mesh covers {engine._src_mesh.size} ranks, but there "
|
||||
f"are {init_info.num_trainer_ranks} trainer ranks"
|
||||
)
|
||||
|
||||
# Validate the whole plan before any rendezvous. A layout m2n cannot
|
||||
# express is a mismatched collective at send time, which hangs rather
|
||||
# than raises -- so every precondition is checked while an exception can
|
||||
# still propagate to the caller.
|
||||
engine._metas = cast(list[M2NParamMeta], source.metadata())
|
||||
for meta in engine._metas:
|
||||
check_transferable(meta.name, meta.dtype, meta.shape)
|
||||
mesh, placements = resolve_layout(engine._src_mesh, meta.placements)
|
||||
validate_layout(mesh, placements, meta.shape, "source")
|
||||
|
||||
rendezvous: Future | None = None
|
||||
if engine.is_controller:
|
||||
engine._executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="m2n-weight-sync"
|
||||
)
|
||||
# Only trainer rank 0 starts the control-plane RPC. Workers join
|
||||
# the communicator inside it, so submit it first, join below, and
|
||||
# wait for it at the end.
|
||||
rendezvous = engine._executor.submit(
|
||||
client.init_weight_transfer_engine,
|
||||
engine._worker_init_info(init_info),
|
||||
)
|
||||
|
||||
# Every trainer rank joins -- reshard is collective over the whole
|
||||
# communicator. Trainer ranks take
|
||||
# [0, num_trainer_ranks) and the workers follow, which is the
|
||||
# contiguous-interval layout m2n requires of both meshes.
|
||||
unique_id_bytes = init_info.nccl_unique_id_bytes
|
||||
if unique_id_bytes is not None:
|
||||
engine.group = uid_init_process_group(
|
||||
unique_id_bytes,
|
||||
rank=init_info.rank,
|
||||
world_size=init_info.world_size,
|
||||
device=torch.accelerator.current_device_index(),
|
||||
)
|
||||
else:
|
||||
assert init_info.master_address is not None
|
||||
assert init_info.master_port is not None
|
||||
engine.group = trainer_init(
|
||||
{
|
||||
"master_address": init_info.master_address,
|
||||
"master_port": init_info.master_port,
|
||||
"world_size": init_info.world_size,
|
||||
},
|
||||
rank=init_info.rank,
|
||||
)
|
||||
|
||||
# The trainer declares the inference topology; the worker uses exactly
|
||||
# this, so neither side has to infer the other's factorization.
|
||||
engine._dst_mesh = M2NMesh(
|
||||
init_info.destination_mesh_dims, init_info.num_trainer_ranks
|
||||
)
|
||||
engine._dst_placements = engine._receive_destination_plan(
|
||||
init_info.num_trainer_ranks
|
||||
)
|
||||
logger.info(
|
||||
"nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), "
|
||||
"inference ranks [%d, %d)",
|
||||
len(engine._metas),
|
||||
init_info.num_trainer_ranks,
|
||||
init_info.num_trainer_ranks,
|
||||
init_info.world_size,
|
||||
)
|
||||
|
||||
# Created after the group so a failed rendezvous does not leave a live
|
||||
# handle behind; shutdown() releases it once caller streams are synced.
|
||||
engine._handle = engine._m2n.Handle.create(
|
||||
engine._m2n.Config(max_cta=init_info.max_cta)
|
||||
)
|
||||
|
||||
# Now safe to collect: both sides have joined, so the RPC has returned
|
||||
# or raised. Doing this last means a worker-side init failure surfaces
|
||||
# here, as an exception from trainer_init, rather than at the first send.
|
||||
if rendezvous is not None:
|
||||
rendezvous.result()
|
||||
return engine
|
||||
|
||||
def _receive_destination_plan(
|
||||
self, first_worker_rank: int
|
||||
) -> list[Placements | None]:
|
||||
"""Receive and validate the inference-side per-parameter placements."""
|
||||
if self.group is None or self._dst_mesh is None:
|
||||
raise RuntimeError(
|
||||
"destination plan requires the communicator and destination mesh"
|
||||
)
|
||||
|
||||
# The first inference worker publishes one placement per parameter.
|
||||
# Every trainer rank receives the complete plan in this single,
|
||||
# initialization-time broadcast.
|
||||
placements = publish_destination_placements(
|
||||
self.group, first_worker_rank, None, len(self._metas)
|
||||
)
|
||||
# The send loop matches placements to metadata by position, so reject
|
||||
# an incomplete or oversized plan before entering any M2N collective.
|
||||
if len(placements) != len(self._metas):
|
||||
raise RuntimeError(
|
||||
f"inference side planned {len(placements)} parameters, but "
|
||||
f"this trainer announced {len(self._metas)}"
|
||||
)
|
||||
# Check every worker-selected destination against the corresponding
|
||||
# global parameter shape before the first transfer.
|
||||
for meta, placement in zip(self._metas, placements):
|
||||
mesh, resolved = resolve_layout(self._dst_mesh, placement)
|
||||
validate_layout(mesh, resolved, meta.shape, "destination")
|
||||
return placements
|
||||
|
||||
def _worker_init_info(self, init_info: M2NTrainerInitInfo) -> dict[str, Any]:
|
||||
"""Handshake payload: rendezvous, both meshes, and the transfer plan."""
|
||||
assert self._src_mesh is not None
|
||||
payload: dict[str, Any] = {
|
||||
"master_address": init_info.master_address,
|
||||
"master_port": init_info.master_port,
|
||||
"nccl_unique_id_b64": init_info.nccl_unique_id_b64,
|
||||
"rank_offset": init_info.num_trainer_ranks,
|
||||
"world_size": init_info.world_size,
|
||||
"src_mesh_dims": list(self._src_mesh.dims),
|
||||
"dst_mesh_dims": list(init_info.destination_mesh_dims),
|
||||
"names": [m.name for m in self._metas],
|
||||
"dtype_names": [_dtype_name(m.dtype) for m in self._metas],
|
||||
"shapes": [list(m.shape) for m in self._metas],
|
||||
"src_placements": [
|
||||
None if m.placements is None else list(m.placements)
|
||||
for m in self._metas
|
||||
],
|
||||
"max_cta": init_info.max_cta,
|
||||
}
|
||||
return {key: value for key, value in payload.items() if value is not None}
|
||||
|
||||
def send_weights(self) -> None:
|
||||
"""Drive one update round: start, reshard concurrently, then finish."""
|
||||
if self._handle is None or self.group is None:
|
||||
raise RuntimeError("nccl_m2n trainer engine is not initialized")
|
||||
|
||||
update: Future | None = None
|
||||
if self.is_controller:
|
||||
self.client.start_weight_update()
|
||||
# The workers reshard from inside `update_weights`, concurrently
|
||||
# with the sends below.
|
||||
assert self._executor is not None
|
||||
update = self._executor.submit(
|
||||
self.client.update_weights,
|
||||
{"names": [m.name for m in self._metas]},
|
||||
)
|
||||
|
||||
try:
|
||||
self._send()
|
||||
finally:
|
||||
if update is not None:
|
||||
update.result()
|
||||
|
||||
if self.is_controller:
|
||||
self.client.finish_weight_update()
|
||||
|
||||
def _send(self) -> None:
|
||||
"""Reshard each local shard into its worker-planned destination."""
|
||||
assert self.group is not None
|
||||
assert self._src_mesh is not None
|
||||
assert self._dst_mesh is not None
|
||||
m2n = self._m2n
|
||||
comm = comm_ptr(self.group)
|
||||
stream = torch.cuda.current_stream()
|
||||
|
||||
for index, (name, tensor) in enumerate(self.source):
|
||||
meta = self._metas[index]
|
||||
if name != meta.name:
|
||||
raise RuntimeError(
|
||||
f"weight source yielded '{name}' at position {index}, but "
|
||||
f"its metadata declared '{meta.name}'; the source must "
|
||||
"iterate in a stable order"
|
||||
)
|
||||
shard = tensor.detach().contiguous()
|
||||
src_mesh, src_placements = resolve_layout(self._src_mesh, meta.placements)
|
||||
dst_mesh, dst_placements = resolve_layout(
|
||||
self._dst_mesh, self._dst_placements[index]
|
||||
)
|
||||
m2n.reshard(
|
||||
shard,
|
||||
None,
|
||||
comm,
|
||||
stream,
|
||||
src_mesh=to_mesh(m2n, src_mesh),
|
||||
src_placements=to_placements(m2n, src_placements),
|
||||
dst_mesh=to_mesh(m2n, dst_mesh),
|
||||
dst_placements=to_placements(m2n, dst_placements),
|
||||
handle=self._handle,
|
||||
)
|
||||
# A custom source may yield temporary storage allocated on another
|
||||
# stream. M2N retains only its device pointer for asynchronous work,
|
||||
# so tell the allocator not to reuse that storage until this stream
|
||||
# has finished with it. For storage allocated on this stream,
|
||||
# record_stream is effectively a no-op.
|
||||
shard.record_stream(stream)
|
||||
|
||||
# As in the other weight-transfer backends, wait before returning
|
||||
# rather than relying on same-stream ordering. The caller may then
|
||||
# safely mutate parameters or continue work on another stream.
|
||||
stream.synchronize()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Destroy the m2n handle, join the helper thread, and drop the group."""
|
||||
if self._handle is not None:
|
||||
torch.accelerator.synchronize()
|
||||
self._handle.destroy()
|
||||
self._handle = None
|
||||
if self._executor is not None:
|
||||
self._executor.shutdown()
|
||||
self._executor = None
|
||||
self.group = None
|
||||
@@ -220,11 +220,11 @@ def worker_init_process_group(
|
||||
|
||||
def trainer_init(
|
||||
init_info: NCCLRendezvous | dict,
|
||||
rank: int = 0,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Initialize NCCL process group for trainer-side weight transfer.
|
||||
|
||||
The trainer is always rank 0 in the process group. Uses the current
|
||||
CUDA device (torch.accelerator.current_device_index()).
|
||||
Uses the current CUDA device (torch.accelerator.current_device_index()).
|
||||
|
||||
Args:
|
||||
init_info: Any object carrying the `NCCLRendezvous` fields (a trainer or
|
||||
@@ -232,6 +232,9 @@ def trainer_init(
|
||||
- master_address: str
|
||||
- master_port: int
|
||||
- world_size: int
|
||||
rank: This trainer process's rank in the group. The broadcast backends
|
||||
have a single trainer at rank 0; multi-rank trainers (m2n) occupy
|
||||
ranks `[0, num_trainer_ranks)` and the workers start after them.
|
||||
|
||||
Returns:
|
||||
PyNcclCommunicator for weight transfer.
|
||||
@@ -246,12 +249,11 @@ def trainer_init(
|
||||
master_port = init_info.master_port
|
||||
world_size = init_info.world_size
|
||||
|
||||
# Trainer is always rank 0
|
||||
device = torch.accelerator.current_device_index()
|
||||
return stateless_init_process_group(
|
||||
master_address,
|
||||
master_port,
|
||||
0,
|
||||
rank,
|
||||
world_size,
|
||||
device,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user