Files
6b6a6c5c98 [RL] Support per-parameter source layouts for NCCL M2N (#60934)
Signed-off-by: pkousha <pkousha@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
2026-10-16 21:41:17 -04:00

865 lines
31 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared helpers for the NCCL M2N (`nccl_m2n`) weight transfer backend.
M2N reshards a tensor between two disjoint meshes of ranks that live in one
communicator: the trainer occupies ranks `[0, T)` and the inference workers
`[T, T + N)`. This module holds everything both sides need — the optional
runtime import, layout descriptors, and the conversion to `nccl.m2n` types —
so the engine module stays about the transfer itself.
"""
import hashlib
import json
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
import torch
from vllm.distributed.weight_transfer.base import ParamMeta
from vllm.utils.nccl import unpinned_nccl_env
if TYPE_CHECKING:
from vllm.distributed.utils import StatelessProcessGroup
# Placement code for a mesh axis that replicates. Any other (non-negative) code
# is the tensor dim that axis shards. Ints rather than `nccl.m2n` objects so
# layouts survive the JSON init handshake without a custom encoder.
REPLICATE = -1
# `NCCL_RESHARD_MAX_TENSOR_DIMS`; 4-D and higher are rejected by the library.
MAX_TENSOR_DIMS = 3
MESH_NDIMS = 2
# Destination mesh convention. Keeping these roles named avoids scattering
# positional axis assumptions through the worker-side planner.
DESTINATION_REPLICA_AXIS = 0
DESTINATION_SHARD_AXIS = 1
# Conservative capacities accepted by the supported nccl_m2n algorithms. They
# are not exposed by its Python API yet; remove these mirrors once nccl_m2n
# provides communication-free plan validation.
MAX_SOURCE_SHARDS = 16
MAX_DEST_SHARDS = 64
# Increment when the JSON init payload changes incompatibly.
M2N_WIRE_SCHEMA_VERSION = 2
SUPPORTED_DTYPES: frozenset[torch.dtype] = frozenset(
dtype
for dtype in (
getattr(torch, name, None)
for name in (
"int8",
"uint8",
"float8_e4m3fn",
"float8_e5m2",
"float16",
"bfloat16",
"int32",
"uint32",
"float32",
"int64",
"uint64",
"float64",
)
)
if dtype is not None
)
_IMPORT_HINT = (
"The nccl_m2n weight transfer backend requires the `nccl-extensions` "
"package (and its `nccl4py` dependency), which is not installed by vLLM. "
"See https://github.com/NVIDIA/nccl-extensions for build "
"and install instructions. It needs NCCL 2.30.5 or newer."
)
def prepare_m2n_local_runtime(m2n: Any, max_cta: int | None) -> tuple[Any, int, Any]:
"""Prepare rank-local resources before any rank enters NCCL init."""
import nccl.core as nccl
device = torch.accelerator.current_device_index()
if not isinstance(device, int) or isinstance(device, bool) or device < 0:
raise RuntimeError(f"invalid current accelerator device index: {device!r}")
handle = m2n.Handle.create(m2n.Config(max_cta=max_cta))
return nccl, device, handle
def create_m2n_communicator(
nccl: Any,
group: "StatelessProcessGroup",
unique_id_bytes: bytes | None,
) -> Any:
"""Create the M2N data-plane communicator through NCCL4Py."""
if unique_id_bytes is None:
envelope: dict[str, bytes | str | None] | None = None
if group.rank == 0:
try:
envelope = {
"unique_id": bytes(nccl.get_unique_id().as_bytes),
"error": None,
}
except Exception as exc:
envelope = {
"unique_id": None,
"error": f"{type(exc).__name__}: {exc}",
}
envelope = group.broadcast_obj(envelope, src=0)
if not isinstance(envelope, Mapping) or set(envelope) != {"unique_id", "error"}:
raise RuntimeError("invalid NCCL4Py unique-id envelope")
error = envelope["error"]
received_unique_id = envelope["unique_id"]
if error is not None:
if not isinstance(error, str) or not error:
raise RuntimeError("invalid NCCL4Py unique-id error")
raise RuntimeError(f"NCCL4Py unique-id generation failed: {error}")
if not isinstance(received_unique_id, bytes):
raise RuntimeError("NCCL4Py unique-id broadcast did not return bytes")
unique_id_bytes = received_unique_id
unique_id = nccl.UniqueId.from_bytes(unique_id_bytes)
with unpinned_nccl_env():
return nccl.Communicator.init(
nranks=group.world_size,
rank=group.rank,
unique_id=unique_id,
)
def import_m2n() -> Any:
"""Import `nccl.m2n` lazily, with an actionable error when it is missing.
Deferred so that importing vLLM — or any other weight transfer backend —
never requires the m2n runtime to be present.
"""
try:
import nccl.m2n as m2n
except ImportError as e:
raise ImportError(f"{_IMPORT_HINT} (import failed: {e})") from e
return m2n
@dataclass(frozen=True)
class M2NMesh:
"""One side's rank topology — pure topology, no tensor placement.
Mirrors `ncclMesh_t`: a 2-axis mesh owning the contiguous rank interval
`[start_rank, start_rank + dims[0] * dims[1])`. There is no 1-D mesh; a
single-axis topology is spelled with a second axis of size 1.
A model may use a different factorization for each tensor. For example,
dense weights can use a DP x TP mesh while expert weights use EDP x EP.
"""
dims: tuple[int, int]
start_rank: int
def __post_init__(self) -> None:
object.__setattr__(self, "dims", tuple(self.dims))
if len(self.dims) != MESH_NDIMS or any(
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
for dim in self.dims
):
raise ValueError(f"mesh dims must be {MESH_NDIMS} positive ints")
if (
not isinstance(self.start_rank, int)
or isinstance(self.start_rank, bool)
or self.start_rank < 0
):
raise ValueError(
f"mesh start_rank must be non-negative, got {self.start_rank}"
)
@property
def size(self) -> int:
return self.dims[0] * self.dims[1]
# A tensor's placement over its side's mesh: one code per mesh axis
# (`REPLICATE`, or the tensor dim that axis shards), or `REPLICATED` for a
# tensor every rank holds in full.
Placements = tuple[int, int]
REPLICATED: Placements | None = None
@dataclass(frozen=True)
class M2NLayout:
"""A tensor's mesh and placement on one side of a transfer."""
mesh: M2NMesh
placements: Placements | None
def __post_init__(self) -> None:
if not isinstance(self.mesh, M2NMesh):
raise TypeError("M2N layout mesh must be an M2NMesh")
if self.placements is not None:
object.__setattr__(self, "placements", tuple(self.placements))
def check_placements(placements: Placements, context: str = "placements") -> None:
"""Reject placement pairs nccl_m2n cannot express."""
if len(placements) != MESH_NDIMS:
raise ValueError(f"{context} must have {MESH_NDIMS} entries")
if any(not isinstance(code, int) or isinstance(code, bool) for code in placements):
raise TypeError(f"{context} must contain integer placement codes")
invalid = [code for code in placements if code < REPLICATE]
if invalid:
raise ValueError(
f"{context} contains invalid placement code {invalid[0]}; valid "
f"codes are {REPLICATE} (Replicate) and non-negative tensor dimensions"
)
num_sharded = sum(code != REPLICATE for code in placements)
if num_sharded == 0:
raise ValueError(
f"{context}: a fully replicated tensor is carried as REPLICATED, "
"not as two REPLICATE axes"
)
if num_sharded == MESH_NDIMS:
raise ValueError(
f"{context}: nccl_m2n needs one REPLICATE mesh axis, but "
f"{placements} shards both"
)
def resolve_layout(
mesh: M2NMesh,
placements: Placements | None,
context: str = "placements",
) -> tuple[M2NMesh, Placements]:
"""Pair one tensor's placement with the mesh m2n should see for it.
A replicated tensor needs a size-1 mesh axis to carry a no-op shard, since
m2n has no `{REPLICATE, REPLICATE}`. It is therefore described over the
*same rank interval* re-factored as `(size, 1)`. That re-factoring is sound
precisely because replication is order-independent — every rank holds the
whole tensor, so it does not matter that `(a, b)` and `(size, 1)` walk the
interval in a different order. A sharded tensor keeps its side's own
factorization, where rank order decides who owns which shard.
"""
if placements is None:
return M2NMesh((mesh.size, 1), mesh.start_rank), (REPLICATE, 0)
check_placements(placements, context)
return mesh, placements
def validate_layout(
mesh: M2NMesh,
placements: Placements,
shape: Sequence[int],
context: str,
) -> tuple[int, ...]:
"""Validate one resolved layout and return its required local shape."""
local_shape = list(shape)
for axis, tensor_dim in enumerate(placements):
if tensor_dim == REPLICATE:
continue
if tensor_dim >= len(shape):
raise ValueError(
f"{context} shards tensor dim {tensor_dim}, but parameter has "
f"rank {len(shape)}"
)
factor = mesh.dims[axis]
if shape[tensor_dim] % factor:
raise ValueError(
f"{context} shards dim {tensor_dim} (size {shape[tensor_dim]}) "
f"over {factor} ranks, which does not divide evenly"
)
local_shape[tensor_dim] //= factor
return tuple(local_shape)
def expected_local_shape(meta: "M2NParamMeta") -> tuple[int, ...]:
"""Validate a parameter's source layout and return its local shape."""
mesh, placements = resolve_layout(
meta.source_layout.mesh,
meta.source_layout.placements,
f"parameter '{meta.name}' source placements",
)
return validate_layout(
mesh, placements, meta.shape, f"parameter '{meta.name}' source layout"
)
def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> None:
"""Reject parameters nccl_m2n cannot transfer."""
if dtype not in SUPPORTED_DTYPES:
raise ValueError(
f"parameter '{name}' has dtype {dtype}, which nccl_m2n does not "
f"support. Supported: {sorted(str(d) for d in SUPPORTED_DTYPES)}"
)
if not 1 <= len(shape) <= MAX_TENSOR_DIMS:
raise ValueError(
f"parameter '{name}' has rank {len(shape)}; nccl_m2n supports "
f"rank 1..{MAX_TENSOR_DIMS}"
)
def _shard_spec(mesh: M2NMesh, placements: Placements) -> tuple[int, int]:
return next(
(code, mesh.dims[axis])
for axis, code in enumerate(placements)
if code != REPLICATE
)
def _max_partition_overlaps(
extent: int,
partitions: int,
other_partitions: int,
) -> int:
"""Return the maximum other partitions intersected by any one partition."""
if extent == 0:
return 0
width = extent // partitions
other_width = extent // other_partitions
max_overlaps = 0
for index in range(partitions):
start = index * width
end = start + width
first_other = start // other_width
last_other = (end - 1) // other_width
max_overlaps = max(max_overlaps, last_other - first_other + 1)
return max_overlaps
def check_plan_limits(
src: tuple[M2NMesh, Placements],
dst: tuple[M2NMesh, Placements],
name: str,
shape: Sequence[int],
) -> None:
"""Reject plans whose per-shard fan-in or fan-out exceeds M2N capacity."""
src_dim, src_shards = _shard_spec(*src)
dst_dim, dst_shards = _shard_spec(*dst)
if any(extent == 0 for extent in shape):
max_sources = max_targets = 0
elif src_dim != dst_dim:
max_sources = src_shards
max_targets = dst_shards
else:
extent = shape[src_dim]
max_sources = _max_partition_overlaps(extent, dst_shards, src_shards)
max_targets = _max_partition_overlaps(extent, src_shards, dst_shards)
if max_sources > MAX_SOURCE_SHARDS:
raise ValueError(
f"parameter '{name}' would feed up to {max_sources} source shards "
f"into one destination shard, over nccl_m2n MAX_SOURCES="
f"{MAX_SOURCE_SHARDS}"
)
if max_targets > MAX_DEST_SHARDS:
raise ValueError(
f"parameter '{name}' would fan one source shard out to up to "
f"{max_targets} destination shards, over nccl_m2n "
f"MAX_TARGETS={MAX_DEST_SHARDS}"
)
def validate_transfer_plan(
metas: Sequence["M2NParamMeta"],
dst_mesh: M2NMesh,
dst_placements: Sequence[Placements | None],
) -> None:
"""Validate the complete ordered source-to-destination transfer plan."""
if len(dst_placements) != len(metas):
raise ValueError(
f"destination plan has {len(dst_placements)} parameters, expected "
f"{len(metas)}"
)
for meta, placement in zip(metas, dst_placements):
check_transferable(meta.name, meta.dtype, meta.shape)
src = resolve_layout(
meta.source_layout.mesh,
meta.source_layout.placements,
f"parameter '{meta.name}' source placements",
)
validate_layout(*src, meta.shape, f"parameter '{meta.name}' source layout")
dst = resolve_layout(
dst_mesh,
placement,
f"parameter '{meta.name}' destination placements",
)
validate_layout(*dst, meta.shape, f"parameter '{meta.name}' destination layout")
check_plan_limits(src, dst, meta.name, meta.shape)
@dataclass(frozen=True)
class M2NParamMeta(ParamMeta):
"""`ParamMeta` extended with how the trainer places this tensor.
The base class carries only name / dtype / full shape, which is not enough
to plan a reshard. `source_layout` carries both the per-parameter mesh and
its placement on that mesh.
"""
source_layout: M2NLayout
def __post_init__(self) -> None:
if not isinstance(self.source_layout, M2NLayout):
raise TypeError("source_layout must be an M2NLayout")
@dataclass(frozen=True)
class M2NWireParam:
"""JSON-safe source metadata for one parameter in the init handshake."""
name: str
dtype_name: str
shape: tuple[int, ...]
src_mesh_dims: tuple[int, int]
src_placements: Placements | None
def __post_init__(self) -> None:
shape = tuple(self.shape)
dims = tuple(self.src_mesh_dims)
placements = None if self.src_placements is None else tuple(self.src_placements)
object.__setattr__(self, "shape", shape)
object.__setattr__(self, "src_mesh_dims", dims)
object.__setattr__(self, "src_placements", placements)
if not isinstance(self.name, str) or not self.name:
raise ValueError("M2N wire parameter name must not be empty")
if not isinstance(self.dtype_name, str) or not self.dtype_name:
raise ValueError(f"parameter '{self.name}' has an empty wire dtype name")
if not shape or any(
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
for dim in shape
):
raise ValueError(
f"parameter '{self.name}' shape must contain positive integers"
)
M2NMesh(cast(tuple[int, int], dims), 0)
if placements is not None:
check_placements(
cast(Placements, placements),
f"parameter '{self.name}' source placements",
)
@classmethod
def from_dict(cls, value: Mapping[str, Any]) -> "M2NWireParam":
"""Parse one strict wire record from its JSON-decoded mapping."""
fields = {
"name",
"dtype_name",
"shape",
"src_mesh_dims",
"src_placements",
}
missing = fields - value.keys()
extra = value.keys() - fields
if missing or extra:
details = []
if missing:
details.append(f"missing {sorted(missing)}")
if extra:
details.append(f"unexpected {sorted(extra)}")
raise ValueError(f"invalid M2N wire parameter: {', '.join(details)}")
return cls(**value)
def to_dict(self) -> dict[str, Any]:
"""Serialize this record using only JSON-compatible values."""
return {
"name": self.name,
"dtype_name": self.dtype_name,
"shape": list(self.shape),
"src_mesh_dims": list(self.src_mesh_dims),
"src_placements": (
None if self.src_placements is None else list(self.src_placements)
),
}
def source_plan_digest(params: Sequence[M2NWireParam]) -> str:
"""Return a rank-independent digest of the ordered source wire plan."""
payload = json.dumps(
{
"schema_version": M2N_WIRE_SCHEMA_VERSION,
"params": [param.to_dict() for param in params],
},
sort_keys=True,
separators=(",", ":"),
).encode()
return hashlib.sha256(payload).hexdigest()
def complete_plan_digest(
source_digest: str,
dst_mesh: M2NMesh,
dst_placements: Sequence[Placements | None],
) -> str:
"""Return a digest of the agreed source and destination plan."""
if not isinstance(source_digest, str) or not source_digest:
raise ValueError("source digest must be a non-empty string")
payload = json.dumps(
{
"schema_version": M2N_WIRE_SCHEMA_VERSION,
"source_digest": source_digest,
"destination_mesh": {
"dims": list(dst_mesh.dims),
"start_rank": dst_mesh.start_rank,
},
"destination_placements": [
None if placement is None else list(placement)
for placement in dst_placements
],
},
sort_keys=True,
separators=(",", ":"),
).encode()
return hashlib.sha256(payload).hexdigest()
def check_source_plan_agreement(
group: "StatelessProcessGroup",
params: Sequence[M2NWireParam],
expected_digest: str | None = None,
local_error: str | None = None,
) -> str:
"""Verify every participant entered with the same source plan."""
prepared_error = local_error
digest: str | None = None
declared: str | None = expected_digest
if prepared_error is None:
try:
digest = source_plan_digest(params)
declared = digest if expected_digest is None else expected_digest
if not isinstance(declared, str) or not declared:
raise ValueError("declared source digest must be a non-empty string")
except Exception as exc:
prepared_error = f"{type(exc).__name__}: {exc}"
elif not isinstance(prepared_error, str) or not prepared_error:
prepared_error = "TypeError: local source error must be a non-empty string"
gathered = group.all_gather_obj(
{
"phase": "source",
"digest": digest,
"declared_digest": declared,
"error": prepared_error,
}
)
errors: list[str] = []
expected_fields = {"phase", "digest", "declared_digest", "error"}
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
agreed: list[tuple[int, str, str]] = []
for rank, item in enumerate(gathered):
if not isinstance(item, Mapping) or set(item) != expected_fields:
errors.append(f"rank {rank}: invalid source preflight envelope")
continue
if item["phase"] != "source":
errors.append(f"rank {rank}: invalid source preflight phase")
continue
error = item["error"]
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid source preflight error")
else:
errors.append(f"rank {rank}: {error}")
continue
computed = item["digest"]
claimed = item["declared_digest"]
if (
not isinstance(computed, str)
or not computed
or not isinstance(claimed, str)
or not claimed
):
errors.append(f"rank {rank}: invalid source digest")
continue
agreed.append((rank, computed, claimed))
if errors:
raise RuntimeError(
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
)
canonical = agreed[0][1]
for rank, computed, claimed in agreed:
if computed != claimed:
errors.append(
f"rank {rank}: computed digest {computed} does not match "
f"declared digest {claimed}"
)
if computed != canonical:
errors.append(f"rank {rank}: source digest disagrees with rank 0")
if errors:
raise RuntimeError(
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
)
return canonical
def check_complete_plan_agreement(
group: "StatelessProcessGroup",
source_digest: str,
dst_mesh: M2NMesh,
dst_placements: Sequence[Placements | None],
local_error: str | None = None,
) -> str:
"""Require every rank to accept the same complete transfer plan."""
prepared_error = local_error
digest: str | None = None
if prepared_error is None:
try:
digest = complete_plan_digest(source_digest, dst_mesh, dst_placements)
except Exception as exc:
prepared_error = f"{type(exc).__name__}: {exc}"
elif not isinstance(prepared_error, str) or not prepared_error:
prepared_error = "TypeError: local plan error must be a non-empty string"
gathered = group.all_gather_obj(
{"phase": "complete_plan", "digest": digest, "error": prepared_error}
)
errors: list[str] = []
valid: list[tuple[int, str]] = []
expected_fields = {"phase", "digest", "error"}
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
for rank, item in enumerate(gathered):
if (
not isinstance(item, Mapping)
or set(item) != expected_fields
or item.get("phase") != "complete_plan"
):
errors.append(f"rank {rank}: invalid complete-plan envelope")
continue
error = item["error"]
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid complete-plan error")
else:
errors.append(f"rank {rank}: {error}")
continue
rank_digest = item["digest"]
if not isinstance(rank_digest, str) or not rank_digest:
errors.append(f"rank {rank}: invalid complete-plan digest")
continue
valid.append((rank, rank_digest))
if not errors:
canonical = valid[0][1]
for rank, rank_digest in valid[1:]:
if rank_digest != canonical:
errors.append(
f"rank {rank}: complete-plan digest disagrees with rank 0"
)
if errors:
raise RuntimeError(
"nccl_m2n complete-plan preflight rejected before first reshard: "
+ "; ".join(errors)
)
return valid[0][1]
def check_runtime_ready_agreement(
group: "StatelessProcessGroup", local_error: str | None
) -> None:
"""Require every rank to prepare local resources before NCCL init."""
prepared_error = local_error
if prepared_error is not None and (
not isinstance(prepared_error, str) or not prepared_error
):
prepared_error = "TypeError: local runtime error must be a non-empty string"
gathered = group.all_gather_obj({"phase": "runtime_ready", "error": prepared_error})
errors: list[str] = []
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
for rank, item in enumerate(gathered):
if (
not isinstance(item, Mapping)
or set(item) != {"phase", "error"}
or item.get("phase") != "runtime_ready"
):
errors.append(f"rank {rank}: invalid runtime readiness envelope")
continue
error = item.get("error")
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid runtime readiness error")
else:
errors.append(f"rank {rank}: {error}")
if errors:
raise RuntimeError(
"nccl_m2n local runtime preflight failed before NCCL init: "
+ "; ".join(errors)
)
def check_data_plane_agreement(
group: "StatelessProcessGroup",
unique_id_bytes: bytes | None,
local_error: str | None = None,
) -> None:
"""Agree on the NCCL rendezvous identity before NCCL init."""
prepared_error = local_error
uid_digest: str | None = None
if prepared_error is None:
try:
if unique_id_bytes is not None:
if not isinstance(unique_id_bytes, bytes):
raise TypeError("NCCL unique id must decode to bytes")
uid_digest = hashlib.sha256(unique_id_bytes).hexdigest()
except Exception as exc:
prepared_error = f"{type(exc).__name__}: {exc}"
elif not isinstance(prepared_error, str) or not prepared_error:
prepared_error = "TypeError: local data-plane error must be non-empty"
gathered = group.all_gather_obj(
{
"phase": "data_plane",
"mode": "uid" if unique_id_bytes is not None else "tcp",
"uid_digest": uid_digest,
"error": prepared_error,
}
)
expected_fields = {
"phase",
"mode",
"uid_digest",
"error",
}
errors: list[str] = []
valid: list[tuple[int, str, str | None]] = []
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
for rank, item in enumerate(gathered):
if (
not isinstance(item, Mapping)
or set(item) != expected_fields
or item.get("phase") != "data_plane"
):
errors.append(f"rank {rank}: invalid data-plane preflight envelope")
continue
error = item["error"]
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid data-plane preflight error")
else:
errors.append(f"rank {rank}: {error}")
continue
mode = item["mode"]
uid = item["uid_digest"]
if mode not in {"tcp", "uid"} or (mode == "uid") != isinstance(uid, str):
errors.append(f"rank {rank}: invalid data-plane identity")
continue
valid.append((rank, mode, uid))
if not errors:
_, mode, uid = valid[0]
for rank, other_mode, other_uid in valid[1:]:
if (other_mode, other_uid) != (mode, uid):
errors.append(f"rank {rank}: NCCL data-plane identity disagrees")
if errors:
raise RuntimeError(
"nccl_m2n data-plane preflight rejected before NCCL init: "
+ "; ".join(errors)
)
def validate_local_buffer(m2n: Any, meta: M2NParamMeta, buffer: Any) -> None:
"""Check source dtype/shape while delegating buffer validity to M2N."""
# A raw pointer carries no inspectable metadata. The source owns its
# correctness; _send supplies the declared dtype and local shape to M2N.
if isinstance(buffer, int):
return
mesh, placements = resolve_layout(
meta.source_layout.mesh, meta.source_layout.placements
)
actual = m2n.DistTensor(
buffer,
mesh=to_mesh(m2n, mesh),
placements=to_placements(m2n, placements),
)
declared = m2n.DistTensor(
None,
local_shape=actual.local_shape,
dtype=meta.dtype,
mesh=to_mesh(m2n, mesh),
placements=to_placements(m2n, placements),
)
if actual.dtype != declared.dtype:
raise ValueError(
f"parameter '{meta.name}' source returned dtype {actual.dtype}, "
f"but its metadata declared {declared.dtype}"
)
expected = expected_local_shape(meta)
if actual.local_shape != expected:
raise ValueError(
f"parameter '{meta.name}' source returned local shape "
f"{actual.local_shape}, but layout {meta.source_layout} implies "
f"{expected} from global shape {meta.shape}"
)
def to_mesh(m2n: Any, mesh: M2NMesh) -> Any:
return m2n.Mesh(mesh.dims, start_rank=mesh.start_rank)
def to_placements(m2n: Any, placements: Placements) -> list[Any]:
return [
m2n.Replicate() if code == REPLICATE else m2n.Shard(code) for code in placements
]
def publish_destination_placements(
comm: Any,
device: torch.device,
stream: torch.cuda.Stream,
first_worker_rank: int,
placements: "Sequence[Placements | None] | None",
num_parameters: int,
) -> list[Placements | None]:
"""Share the worker-side destination plan with every rank in the group.
The trainer must issue each reshard with the same destination the workers
use, but once destinations are per-parameter they depend on the inference
model, which only the workers can see. The first worker publishes them here,
over the shared NCCL communicator; trainer ranks pass `None` and receive
them, and the other workers pass their own so a disagreement is caught
rather than deadlocking later.
Only placements travel: the destination *mesh* is still derived from the
rank split, identically on both sides. The plan is published on the data
plane before any M2N reshard collective.
"""
replicated_sentinel = REPLICATE - 1
encoded = torch.full(
(num_parameters, MESH_NDIMS),
replicated_sentinel,
dtype=torch.int8,
device=device,
)
if comm.rank == first_worker_rank:
if placements is None or len(placements) != num_parameters:
raise ValueError(
f"publishing rank needs {num_parameters} destination placements"
)
rows = [
[replicated_sentinel] * MESH_NDIMS if placement is REPLICATED else placement
for placement in placements
]
if rows:
encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=device))
comm.broadcast(encoded, encoded, root=first_worker_rank, stream=stream)
return [
REPLICATED
if all(code == replicated_sentinel for code in row)
else cast(Placements, tuple(row))
for row in encoded.tolist()
]