Signed-off-by: pkousha <pkousha@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
865 lines
31 KiB
Python
865 lines
31 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Shared helpers for the NCCL M2N (`nccl_m2n`) weight transfer backend.
|
|
|
|
M2N reshards a tensor between two disjoint meshes of ranks that live in one
|
|
communicator: the trainer occupies ranks `[0, T)` and the inference workers
|
|
`[T, T + N)`. This module holds everything both sides need — the optional
|
|
runtime import, layout descriptors, and the conversion to `nccl.m2n` types —
|
|
so the engine module stays about the transfer itself.
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
from collections.abc import Mapping, Sequence
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING, Any, cast
|
|
|
|
import torch
|
|
|
|
from vllm.distributed.weight_transfer.base import ParamMeta
|
|
from vllm.utils.nccl import unpinned_nccl_env
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.distributed.utils import StatelessProcessGroup
|
|
|
|
# Placement code for a mesh axis that replicates. Any other (non-negative) code
|
|
# is the tensor dim that axis shards. Ints rather than `nccl.m2n` objects so
|
|
# layouts survive the JSON init handshake without a custom encoder.
|
|
REPLICATE = -1
|
|
|
|
# `NCCL_RESHARD_MAX_TENSOR_DIMS`; 4-D and higher are rejected by the library.
|
|
MAX_TENSOR_DIMS = 3
|
|
|
|
MESH_NDIMS = 2
|
|
|
|
# Destination mesh convention. Keeping these roles named avoids scattering
|
|
# positional axis assumptions through the worker-side planner.
|
|
DESTINATION_REPLICA_AXIS = 0
|
|
DESTINATION_SHARD_AXIS = 1
|
|
|
|
# Conservative capacities accepted by the supported nccl_m2n algorithms. They
|
|
# are not exposed by its Python API yet; remove these mirrors once nccl_m2n
|
|
# provides communication-free plan validation.
|
|
MAX_SOURCE_SHARDS = 16
|
|
MAX_DEST_SHARDS = 64
|
|
|
|
# Increment when the JSON init payload changes incompatibly.
|
|
M2N_WIRE_SCHEMA_VERSION = 2
|
|
|
|
SUPPORTED_DTYPES: frozenset[torch.dtype] = frozenset(
|
|
dtype
|
|
for dtype in (
|
|
getattr(torch, name, None)
|
|
for name in (
|
|
"int8",
|
|
"uint8",
|
|
"float8_e4m3fn",
|
|
"float8_e5m2",
|
|
"float16",
|
|
"bfloat16",
|
|
"int32",
|
|
"uint32",
|
|
"float32",
|
|
"int64",
|
|
"uint64",
|
|
"float64",
|
|
)
|
|
)
|
|
if dtype is not None
|
|
)
|
|
|
|
_IMPORT_HINT = (
|
|
"The nccl_m2n weight transfer backend requires the `nccl-extensions` "
|
|
"package (and its `nccl4py` dependency), which is not installed by vLLM. "
|
|
"See https://github.com/NVIDIA/nccl-extensions for build "
|
|
"and install instructions. It needs NCCL 2.30.5 or newer."
|
|
)
|
|
|
|
|
|
def prepare_m2n_local_runtime(m2n: Any, max_cta: int | None) -> tuple[Any, int, Any]:
|
|
"""Prepare rank-local resources before any rank enters NCCL init."""
|
|
import nccl.core as nccl
|
|
|
|
device = torch.accelerator.current_device_index()
|
|
if not isinstance(device, int) or isinstance(device, bool) or device < 0:
|
|
raise RuntimeError(f"invalid current accelerator device index: {device!r}")
|
|
handle = m2n.Handle.create(m2n.Config(max_cta=max_cta))
|
|
return nccl, device, handle
|
|
|
|
|
|
def create_m2n_communicator(
|
|
nccl: Any,
|
|
group: "StatelessProcessGroup",
|
|
unique_id_bytes: bytes | None,
|
|
) -> Any:
|
|
"""Create the M2N data-plane communicator through NCCL4Py."""
|
|
if unique_id_bytes is None:
|
|
envelope: dict[str, bytes | str | None] | None = None
|
|
if group.rank == 0:
|
|
try:
|
|
envelope = {
|
|
"unique_id": bytes(nccl.get_unique_id().as_bytes),
|
|
"error": None,
|
|
}
|
|
except Exception as exc:
|
|
envelope = {
|
|
"unique_id": None,
|
|
"error": f"{type(exc).__name__}: {exc}",
|
|
}
|
|
envelope = group.broadcast_obj(envelope, src=0)
|
|
if not isinstance(envelope, Mapping) or set(envelope) != {"unique_id", "error"}:
|
|
raise RuntimeError("invalid NCCL4Py unique-id envelope")
|
|
error = envelope["error"]
|
|
received_unique_id = envelope["unique_id"]
|
|
if error is not None:
|
|
if not isinstance(error, str) or not error:
|
|
raise RuntimeError("invalid NCCL4Py unique-id error")
|
|
raise RuntimeError(f"NCCL4Py unique-id generation failed: {error}")
|
|
if not isinstance(received_unique_id, bytes):
|
|
raise RuntimeError("NCCL4Py unique-id broadcast did not return bytes")
|
|
unique_id_bytes = received_unique_id
|
|
|
|
unique_id = nccl.UniqueId.from_bytes(unique_id_bytes)
|
|
with unpinned_nccl_env():
|
|
return nccl.Communicator.init(
|
|
nranks=group.world_size,
|
|
rank=group.rank,
|
|
unique_id=unique_id,
|
|
)
|
|
|
|
|
|
def import_m2n() -> Any:
|
|
"""Import `nccl.m2n` lazily, with an actionable error when it is missing.
|
|
|
|
Deferred so that importing vLLM — or any other weight transfer backend —
|
|
never requires the m2n runtime to be present.
|
|
"""
|
|
try:
|
|
import nccl.m2n as m2n
|
|
except ImportError as e:
|
|
raise ImportError(f"{_IMPORT_HINT} (import failed: {e})") from e
|
|
return m2n
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class M2NMesh:
|
|
"""One side's rank topology — pure topology, no tensor placement.
|
|
|
|
Mirrors `ncclMesh_t`: a 2-axis mesh owning the contiguous rank interval
|
|
`[start_rank, start_rank + dims[0] * dims[1])`. There is no 1-D mesh; a
|
|
single-axis topology is spelled with a second axis of size 1.
|
|
|
|
A model may use a different factorization for each tensor. For example,
|
|
dense weights can use a DP x TP mesh while expert weights use EDP x EP.
|
|
"""
|
|
|
|
dims: tuple[int, int]
|
|
start_rank: int
|
|
|
|
def __post_init__(self) -> None:
|
|
object.__setattr__(self, "dims", tuple(self.dims))
|
|
if len(self.dims) != MESH_NDIMS or any(
|
|
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
|
|
for dim in self.dims
|
|
):
|
|
raise ValueError(f"mesh dims must be {MESH_NDIMS} positive ints")
|
|
if (
|
|
not isinstance(self.start_rank, int)
|
|
or isinstance(self.start_rank, bool)
|
|
or self.start_rank < 0
|
|
):
|
|
raise ValueError(
|
|
f"mesh start_rank must be non-negative, got {self.start_rank}"
|
|
)
|
|
|
|
@property
|
|
def size(self) -> int:
|
|
return self.dims[0] * self.dims[1]
|
|
|
|
|
|
# A tensor's placement over its side's mesh: one code per mesh axis
|
|
# (`REPLICATE`, or the tensor dim that axis shards), or `REPLICATED` for a
|
|
# tensor every rank holds in full.
|
|
Placements = tuple[int, int]
|
|
REPLICATED: Placements | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class M2NLayout:
|
|
"""A tensor's mesh and placement on one side of a transfer."""
|
|
|
|
mesh: M2NMesh
|
|
placements: Placements | None
|
|
|
|
def __post_init__(self) -> None:
|
|
if not isinstance(self.mesh, M2NMesh):
|
|
raise TypeError("M2N layout mesh must be an M2NMesh")
|
|
if self.placements is not None:
|
|
object.__setattr__(self, "placements", tuple(self.placements))
|
|
|
|
|
|
def check_placements(placements: Placements, context: str = "placements") -> None:
|
|
"""Reject placement pairs nccl_m2n cannot express."""
|
|
if len(placements) != MESH_NDIMS:
|
|
raise ValueError(f"{context} must have {MESH_NDIMS} entries")
|
|
if any(not isinstance(code, int) or isinstance(code, bool) for code in placements):
|
|
raise TypeError(f"{context} must contain integer placement codes")
|
|
invalid = [code for code in placements if code < REPLICATE]
|
|
if invalid:
|
|
raise ValueError(
|
|
f"{context} contains invalid placement code {invalid[0]}; valid "
|
|
f"codes are {REPLICATE} (Replicate) and non-negative tensor dimensions"
|
|
)
|
|
num_sharded = sum(code != REPLICATE for code in placements)
|
|
if num_sharded == 0:
|
|
raise ValueError(
|
|
f"{context}: a fully replicated tensor is carried as REPLICATED, "
|
|
"not as two REPLICATE axes"
|
|
)
|
|
if num_sharded == MESH_NDIMS:
|
|
raise ValueError(
|
|
f"{context}: nccl_m2n needs one REPLICATE mesh axis, but "
|
|
f"{placements} shards both"
|
|
)
|
|
|
|
|
|
def resolve_layout(
|
|
mesh: M2NMesh,
|
|
placements: Placements | None,
|
|
context: str = "placements",
|
|
) -> tuple[M2NMesh, Placements]:
|
|
"""Pair one tensor's placement with the mesh m2n should see for it.
|
|
|
|
A replicated tensor needs a size-1 mesh axis to carry a no-op shard, since
|
|
m2n has no `{REPLICATE, REPLICATE}`. It is therefore described over the
|
|
*same rank interval* re-factored as `(size, 1)`. That re-factoring is sound
|
|
precisely because replication is order-independent — every rank holds the
|
|
whole tensor, so it does not matter that `(a, b)` and `(size, 1)` walk the
|
|
interval in a different order. A sharded tensor keeps its side's own
|
|
factorization, where rank order decides who owns which shard.
|
|
"""
|
|
if placements is None:
|
|
return M2NMesh((mesh.size, 1), mesh.start_rank), (REPLICATE, 0)
|
|
check_placements(placements, context)
|
|
return mesh, placements
|
|
|
|
|
|
def validate_layout(
|
|
mesh: M2NMesh,
|
|
placements: Placements,
|
|
shape: Sequence[int],
|
|
context: str,
|
|
) -> tuple[int, ...]:
|
|
"""Validate one resolved layout and return its required local shape."""
|
|
local_shape = list(shape)
|
|
for axis, tensor_dim in enumerate(placements):
|
|
if tensor_dim == REPLICATE:
|
|
continue
|
|
if tensor_dim >= len(shape):
|
|
raise ValueError(
|
|
f"{context} shards tensor dim {tensor_dim}, but parameter has "
|
|
f"rank {len(shape)}"
|
|
)
|
|
factor = mesh.dims[axis]
|
|
if shape[tensor_dim] % factor:
|
|
raise ValueError(
|
|
f"{context} shards dim {tensor_dim} (size {shape[tensor_dim]}) "
|
|
f"over {factor} ranks, which does not divide evenly"
|
|
)
|
|
local_shape[tensor_dim] //= factor
|
|
return tuple(local_shape)
|
|
|
|
|
|
def expected_local_shape(meta: "M2NParamMeta") -> tuple[int, ...]:
|
|
"""Validate a parameter's source layout and return its local shape."""
|
|
mesh, placements = resolve_layout(
|
|
meta.source_layout.mesh,
|
|
meta.source_layout.placements,
|
|
f"parameter '{meta.name}' source placements",
|
|
)
|
|
return validate_layout(
|
|
mesh, placements, meta.shape, f"parameter '{meta.name}' source layout"
|
|
)
|
|
|
|
|
|
def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> None:
|
|
"""Reject parameters nccl_m2n cannot transfer."""
|
|
if dtype not in SUPPORTED_DTYPES:
|
|
raise ValueError(
|
|
f"parameter '{name}' has dtype {dtype}, which nccl_m2n does not "
|
|
f"support. Supported: {sorted(str(d) for d in SUPPORTED_DTYPES)}"
|
|
)
|
|
if not 1 <= len(shape) <= MAX_TENSOR_DIMS:
|
|
raise ValueError(
|
|
f"parameter '{name}' has rank {len(shape)}; nccl_m2n supports "
|
|
f"rank 1..{MAX_TENSOR_DIMS}"
|
|
)
|
|
|
|
|
|
def _shard_spec(mesh: M2NMesh, placements: Placements) -> tuple[int, int]:
|
|
return next(
|
|
(code, mesh.dims[axis])
|
|
for axis, code in enumerate(placements)
|
|
if code != REPLICATE
|
|
)
|
|
|
|
|
|
def _max_partition_overlaps(
|
|
extent: int,
|
|
partitions: int,
|
|
other_partitions: int,
|
|
) -> int:
|
|
"""Return the maximum other partitions intersected by any one partition."""
|
|
if extent == 0:
|
|
return 0
|
|
width = extent // partitions
|
|
other_width = extent // other_partitions
|
|
max_overlaps = 0
|
|
for index in range(partitions):
|
|
start = index * width
|
|
end = start + width
|
|
first_other = start // other_width
|
|
last_other = (end - 1) // other_width
|
|
max_overlaps = max(max_overlaps, last_other - first_other + 1)
|
|
return max_overlaps
|
|
|
|
|
|
def check_plan_limits(
|
|
src: tuple[M2NMesh, Placements],
|
|
dst: tuple[M2NMesh, Placements],
|
|
name: str,
|
|
shape: Sequence[int],
|
|
) -> None:
|
|
"""Reject plans whose per-shard fan-in or fan-out exceeds M2N capacity."""
|
|
src_dim, src_shards = _shard_spec(*src)
|
|
dst_dim, dst_shards = _shard_spec(*dst)
|
|
if any(extent == 0 for extent in shape):
|
|
max_sources = max_targets = 0
|
|
elif src_dim != dst_dim:
|
|
max_sources = src_shards
|
|
max_targets = dst_shards
|
|
else:
|
|
extent = shape[src_dim]
|
|
max_sources = _max_partition_overlaps(extent, dst_shards, src_shards)
|
|
max_targets = _max_partition_overlaps(extent, src_shards, dst_shards)
|
|
|
|
if max_sources > MAX_SOURCE_SHARDS:
|
|
raise ValueError(
|
|
f"parameter '{name}' would feed up to {max_sources} source shards "
|
|
f"into one destination shard, over nccl_m2n MAX_SOURCES="
|
|
f"{MAX_SOURCE_SHARDS}"
|
|
)
|
|
if max_targets > MAX_DEST_SHARDS:
|
|
raise ValueError(
|
|
f"parameter '{name}' would fan one source shard out to up to "
|
|
f"{max_targets} destination shards, over nccl_m2n "
|
|
f"MAX_TARGETS={MAX_DEST_SHARDS}"
|
|
)
|
|
|
|
|
|
def validate_transfer_plan(
|
|
metas: Sequence["M2NParamMeta"],
|
|
dst_mesh: M2NMesh,
|
|
dst_placements: Sequence[Placements | None],
|
|
) -> None:
|
|
"""Validate the complete ordered source-to-destination transfer plan."""
|
|
if len(dst_placements) != len(metas):
|
|
raise ValueError(
|
|
f"destination plan has {len(dst_placements)} parameters, expected "
|
|
f"{len(metas)}"
|
|
)
|
|
for meta, placement in zip(metas, dst_placements):
|
|
check_transferable(meta.name, meta.dtype, meta.shape)
|
|
src = resolve_layout(
|
|
meta.source_layout.mesh,
|
|
meta.source_layout.placements,
|
|
f"parameter '{meta.name}' source placements",
|
|
)
|
|
validate_layout(*src, meta.shape, f"parameter '{meta.name}' source layout")
|
|
dst = resolve_layout(
|
|
dst_mesh,
|
|
placement,
|
|
f"parameter '{meta.name}' destination placements",
|
|
)
|
|
validate_layout(*dst, meta.shape, f"parameter '{meta.name}' destination layout")
|
|
check_plan_limits(src, dst, meta.name, meta.shape)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class M2NParamMeta(ParamMeta):
|
|
"""`ParamMeta` extended with how the trainer places this tensor.
|
|
|
|
The base class carries only name / dtype / full shape, which is not enough
|
|
to plan a reshard. `source_layout` carries both the per-parameter mesh and
|
|
its placement on that mesh.
|
|
"""
|
|
|
|
source_layout: M2NLayout
|
|
|
|
def __post_init__(self) -> None:
|
|
if not isinstance(self.source_layout, M2NLayout):
|
|
raise TypeError("source_layout must be an M2NLayout")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class M2NWireParam:
|
|
"""JSON-safe source metadata for one parameter in the init handshake."""
|
|
|
|
name: str
|
|
dtype_name: str
|
|
shape: tuple[int, ...]
|
|
src_mesh_dims: tuple[int, int]
|
|
src_placements: Placements | None
|
|
|
|
def __post_init__(self) -> None:
|
|
shape = tuple(self.shape)
|
|
dims = tuple(self.src_mesh_dims)
|
|
placements = None if self.src_placements is None else tuple(self.src_placements)
|
|
object.__setattr__(self, "shape", shape)
|
|
object.__setattr__(self, "src_mesh_dims", dims)
|
|
object.__setattr__(self, "src_placements", placements)
|
|
if not isinstance(self.name, str) or not self.name:
|
|
raise ValueError("M2N wire parameter name must not be empty")
|
|
if not isinstance(self.dtype_name, str) or not self.dtype_name:
|
|
raise ValueError(f"parameter '{self.name}' has an empty wire dtype name")
|
|
if not shape or any(
|
|
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
|
|
for dim in shape
|
|
):
|
|
raise ValueError(
|
|
f"parameter '{self.name}' shape must contain positive integers"
|
|
)
|
|
M2NMesh(cast(tuple[int, int], dims), 0)
|
|
if placements is not None:
|
|
check_placements(
|
|
cast(Placements, placements),
|
|
f"parameter '{self.name}' source placements",
|
|
)
|
|
|
|
@classmethod
|
|
def from_dict(cls, value: Mapping[str, Any]) -> "M2NWireParam":
|
|
"""Parse one strict wire record from its JSON-decoded mapping."""
|
|
fields = {
|
|
"name",
|
|
"dtype_name",
|
|
"shape",
|
|
"src_mesh_dims",
|
|
"src_placements",
|
|
}
|
|
missing = fields - value.keys()
|
|
extra = value.keys() - fields
|
|
if missing or extra:
|
|
details = []
|
|
if missing:
|
|
details.append(f"missing {sorted(missing)}")
|
|
if extra:
|
|
details.append(f"unexpected {sorted(extra)}")
|
|
raise ValueError(f"invalid M2N wire parameter: {', '.join(details)}")
|
|
return cls(**value)
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
"""Serialize this record using only JSON-compatible values."""
|
|
return {
|
|
"name": self.name,
|
|
"dtype_name": self.dtype_name,
|
|
"shape": list(self.shape),
|
|
"src_mesh_dims": list(self.src_mesh_dims),
|
|
"src_placements": (
|
|
None if self.src_placements is None else list(self.src_placements)
|
|
),
|
|
}
|
|
|
|
|
|
def source_plan_digest(params: Sequence[M2NWireParam]) -> str:
|
|
"""Return a rank-independent digest of the ordered source wire plan."""
|
|
payload = json.dumps(
|
|
{
|
|
"schema_version": M2N_WIRE_SCHEMA_VERSION,
|
|
"params": [param.to_dict() for param in params],
|
|
},
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode()
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
|
|
def complete_plan_digest(
|
|
source_digest: str,
|
|
dst_mesh: M2NMesh,
|
|
dst_placements: Sequence[Placements | None],
|
|
) -> str:
|
|
"""Return a digest of the agreed source and destination plan."""
|
|
if not isinstance(source_digest, str) or not source_digest:
|
|
raise ValueError("source digest must be a non-empty string")
|
|
payload = json.dumps(
|
|
{
|
|
"schema_version": M2N_WIRE_SCHEMA_VERSION,
|
|
"source_digest": source_digest,
|
|
"destination_mesh": {
|
|
"dims": list(dst_mesh.dims),
|
|
"start_rank": dst_mesh.start_rank,
|
|
},
|
|
"destination_placements": [
|
|
None if placement is None else list(placement)
|
|
for placement in dst_placements
|
|
],
|
|
},
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode()
|
|
return hashlib.sha256(payload).hexdigest()
|
|
|
|
|
|
def check_source_plan_agreement(
|
|
group: "StatelessProcessGroup",
|
|
params: Sequence[M2NWireParam],
|
|
expected_digest: str | None = None,
|
|
local_error: str | None = None,
|
|
) -> str:
|
|
"""Verify every participant entered with the same source plan."""
|
|
prepared_error = local_error
|
|
digest: str | None = None
|
|
declared: str | None = expected_digest
|
|
if prepared_error is None:
|
|
try:
|
|
digest = source_plan_digest(params)
|
|
declared = digest if expected_digest is None else expected_digest
|
|
if not isinstance(declared, str) or not declared:
|
|
raise ValueError("declared source digest must be a non-empty string")
|
|
except Exception as exc:
|
|
prepared_error = f"{type(exc).__name__}: {exc}"
|
|
elif not isinstance(prepared_error, str) or not prepared_error:
|
|
prepared_error = "TypeError: local source error must be a non-empty string"
|
|
|
|
gathered = group.all_gather_obj(
|
|
{
|
|
"phase": "source",
|
|
"digest": digest,
|
|
"declared_digest": declared,
|
|
"error": prepared_error,
|
|
}
|
|
)
|
|
errors: list[str] = []
|
|
expected_fields = {"phase", "digest", "declared_digest", "error"}
|
|
if len(gathered) != group.world_size:
|
|
errors.append(
|
|
"control-plane all-gather returned "
|
|
f"{len(gathered)} records for world size {group.world_size}"
|
|
)
|
|
agreed: list[tuple[int, str, str]] = []
|
|
for rank, item in enumerate(gathered):
|
|
if not isinstance(item, Mapping) or set(item) != expected_fields:
|
|
errors.append(f"rank {rank}: invalid source preflight envelope")
|
|
continue
|
|
if item["phase"] != "source":
|
|
errors.append(f"rank {rank}: invalid source preflight phase")
|
|
continue
|
|
error = item["error"]
|
|
if error is not None:
|
|
if not isinstance(error, str) or not error:
|
|
errors.append(f"rank {rank}: invalid source preflight error")
|
|
else:
|
|
errors.append(f"rank {rank}: {error}")
|
|
continue
|
|
computed = item["digest"]
|
|
claimed = item["declared_digest"]
|
|
if (
|
|
not isinstance(computed, str)
|
|
or not computed
|
|
or not isinstance(claimed, str)
|
|
or not claimed
|
|
):
|
|
errors.append(f"rank {rank}: invalid source digest")
|
|
continue
|
|
agreed.append((rank, computed, claimed))
|
|
if errors:
|
|
raise RuntimeError(
|
|
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
|
|
)
|
|
|
|
canonical = agreed[0][1]
|
|
for rank, computed, claimed in agreed:
|
|
if computed != claimed:
|
|
errors.append(
|
|
f"rank {rank}: computed digest {computed} does not match "
|
|
f"declared digest {claimed}"
|
|
)
|
|
if computed != canonical:
|
|
errors.append(f"rank {rank}: source digest disagrees with rank 0")
|
|
if errors:
|
|
raise RuntimeError(
|
|
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
|
|
)
|
|
return canonical
|
|
|
|
|
|
def check_complete_plan_agreement(
|
|
group: "StatelessProcessGroup",
|
|
source_digest: str,
|
|
dst_mesh: M2NMesh,
|
|
dst_placements: Sequence[Placements | None],
|
|
local_error: str | None = None,
|
|
) -> str:
|
|
"""Require every rank to accept the same complete transfer plan."""
|
|
prepared_error = local_error
|
|
digest: str | None = None
|
|
if prepared_error is None:
|
|
try:
|
|
digest = complete_plan_digest(source_digest, dst_mesh, dst_placements)
|
|
except Exception as exc:
|
|
prepared_error = f"{type(exc).__name__}: {exc}"
|
|
elif not isinstance(prepared_error, str) or not prepared_error:
|
|
prepared_error = "TypeError: local plan error must be a non-empty string"
|
|
|
|
gathered = group.all_gather_obj(
|
|
{"phase": "complete_plan", "digest": digest, "error": prepared_error}
|
|
)
|
|
errors: list[str] = []
|
|
valid: list[tuple[int, str]] = []
|
|
expected_fields = {"phase", "digest", "error"}
|
|
if len(gathered) != group.world_size:
|
|
errors.append(
|
|
"control-plane all-gather returned "
|
|
f"{len(gathered)} records for world size {group.world_size}"
|
|
)
|
|
for rank, item in enumerate(gathered):
|
|
if (
|
|
not isinstance(item, Mapping)
|
|
or set(item) != expected_fields
|
|
or item.get("phase") != "complete_plan"
|
|
):
|
|
errors.append(f"rank {rank}: invalid complete-plan envelope")
|
|
continue
|
|
error = item["error"]
|
|
if error is not None:
|
|
if not isinstance(error, str) or not error:
|
|
errors.append(f"rank {rank}: invalid complete-plan error")
|
|
else:
|
|
errors.append(f"rank {rank}: {error}")
|
|
continue
|
|
rank_digest = item["digest"]
|
|
if not isinstance(rank_digest, str) or not rank_digest:
|
|
errors.append(f"rank {rank}: invalid complete-plan digest")
|
|
continue
|
|
valid.append((rank, rank_digest))
|
|
if not errors:
|
|
canonical = valid[0][1]
|
|
for rank, rank_digest in valid[1:]:
|
|
if rank_digest != canonical:
|
|
errors.append(
|
|
f"rank {rank}: complete-plan digest disagrees with rank 0"
|
|
)
|
|
if errors:
|
|
raise RuntimeError(
|
|
"nccl_m2n complete-plan preflight rejected before first reshard: "
|
|
+ "; ".join(errors)
|
|
)
|
|
return valid[0][1]
|
|
|
|
|
|
def check_runtime_ready_agreement(
|
|
group: "StatelessProcessGroup", local_error: str | None
|
|
) -> None:
|
|
"""Require every rank to prepare local resources before NCCL init."""
|
|
prepared_error = local_error
|
|
if prepared_error is not None and (
|
|
not isinstance(prepared_error, str) or not prepared_error
|
|
):
|
|
prepared_error = "TypeError: local runtime error must be a non-empty string"
|
|
gathered = group.all_gather_obj({"phase": "runtime_ready", "error": prepared_error})
|
|
errors: list[str] = []
|
|
if len(gathered) != group.world_size:
|
|
errors.append(
|
|
"control-plane all-gather returned "
|
|
f"{len(gathered)} records for world size {group.world_size}"
|
|
)
|
|
for rank, item in enumerate(gathered):
|
|
if (
|
|
not isinstance(item, Mapping)
|
|
or set(item) != {"phase", "error"}
|
|
or item.get("phase") != "runtime_ready"
|
|
):
|
|
errors.append(f"rank {rank}: invalid runtime readiness envelope")
|
|
continue
|
|
error = item.get("error")
|
|
if error is not None:
|
|
if not isinstance(error, str) or not error:
|
|
errors.append(f"rank {rank}: invalid runtime readiness error")
|
|
else:
|
|
errors.append(f"rank {rank}: {error}")
|
|
if errors:
|
|
raise RuntimeError(
|
|
"nccl_m2n local runtime preflight failed before NCCL init: "
|
|
+ "; ".join(errors)
|
|
)
|
|
|
|
|
|
def check_data_plane_agreement(
|
|
group: "StatelessProcessGroup",
|
|
unique_id_bytes: bytes | None,
|
|
local_error: str | None = None,
|
|
) -> None:
|
|
"""Agree on the NCCL rendezvous identity before NCCL init."""
|
|
prepared_error = local_error
|
|
uid_digest: str | None = None
|
|
if prepared_error is None:
|
|
try:
|
|
if unique_id_bytes is not None:
|
|
if not isinstance(unique_id_bytes, bytes):
|
|
raise TypeError("NCCL unique id must decode to bytes")
|
|
uid_digest = hashlib.sha256(unique_id_bytes).hexdigest()
|
|
except Exception as exc:
|
|
prepared_error = f"{type(exc).__name__}: {exc}"
|
|
elif not isinstance(prepared_error, str) or not prepared_error:
|
|
prepared_error = "TypeError: local data-plane error must be non-empty"
|
|
|
|
gathered = group.all_gather_obj(
|
|
{
|
|
"phase": "data_plane",
|
|
"mode": "uid" if unique_id_bytes is not None else "tcp",
|
|
"uid_digest": uid_digest,
|
|
"error": prepared_error,
|
|
}
|
|
)
|
|
expected_fields = {
|
|
"phase",
|
|
"mode",
|
|
"uid_digest",
|
|
"error",
|
|
}
|
|
errors: list[str] = []
|
|
valid: list[tuple[int, str, str | None]] = []
|
|
if len(gathered) != group.world_size:
|
|
errors.append(
|
|
"control-plane all-gather returned "
|
|
f"{len(gathered)} records for world size {group.world_size}"
|
|
)
|
|
for rank, item in enumerate(gathered):
|
|
if (
|
|
not isinstance(item, Mapping)
|
|
or set(item) != expected_fields
|
|
or item.get("phase") != "data_plane"
|
|
):
|
|
errors.append(f"rank {rank}: invalid data-plane preflight envelope")
|
|
continue
|
|
error = item["error"]
|
|
if error is not None:
|
|
if not isinstance(error, str) or not error:
|
|
errors.append(f"rank {rank}: invalid data-plane preflight error")
|
|
else:
|
|
errors.append(f"rank {rank}: {error}")
|
|
continue
|
|
mode = item["mode"]
|
|
uid = item["uid_digest"]
|
|
if mode not in {"tcp", "uid"} or (mode == "uid") != isinstance(uid, str):
|
|
errors.append(f"rank {rank}: invalid data-plane identity")
|
|
continue
|
|
valid.append((rank, mode, uid))
|
|
if not errors:
|
|
_, mode, uid = valid[0]
|
|
for rank, other_mode, other_uid in valid[1:]:
|
|
if (other_mode, other_uid) != (mode, uid):
|
|
errors.append(f"rank {rank}: NCCL data-plane identity disagrees")
|
|
if errors:
|
|
raise RuntimeError(
|
|
"nccl_m2n data-plane preflight rejected before NCCL init: "
|
|
+ "; ".join(errors)
|
|
)
|
|
|
|
|
|
def validate_local_buffer(m2n: Any, meta: M2NParamMeta, buffer: Any) -> None:
|
|
"""Check source dtype/shape while delegating buffer validity to M2N."""
|
|
# A raw pointer carries no inspectable metadata. The source owns its
|
|
# correctness; _send supplies the declared dtype and local shape to M2N.
|
|
if isinstance(buffer, int):
|
|
return
|
|
|
|
mesh, placements = resolve_layout(
|
|
meta.source_layout.mesh, meta.source_layout.placements
|
|
)
|
|
actual = m2n.DistTensor(
|
|
buffer,
|
|
mesh=to_mesh(m2n, mesh),
|
|
placements=to_placements(m2n, placements),
|
|
)
|
|
declared = m2n.DistTensor(
|
|
None,
|
|
local_shape=actual.local_shape,
|
|
dtype=meta.dtype,
|
|
mesh=to_mesh(m2n, mesh),
|
|
placements=to_placements(m2n, placements),
|
|
)
|
|
if actual.dtype != declared.dtype:
|
|
raise ValueError(
|
|
f"parameter '{meta.name}' source returned dtype {actual.dtype}, "
|
|
f"but its metadata declared {declared.dtype}"
|
|
)
|
|
|
|
expected = expected_local_shape(meta)
|
|
if actual.local_shape != expected:
|
|
raise ValueError(
|
|
f"parameter '{meta.name}' source returned local shape "
|
|
f"{actual.local_shape}, but layout {meta.source_layout} implies "
|
|
f"{expected} from global shape {meta.shape}"
|
|
)
|
|
|
|
|
|
def to_mesh(m2n: Any, mesh: M2NMesh) -> Any:
|
|
return m2n.Mesh(mesh.dims, start_rank=mesh.start_rank)
|
|
|
|
|
|
def to_placements(m2n: Any, placements: Placements) -> list[Any]:
|
|
return [
|
|
m2n.Replicate() if code == REPLICATE else m2n.Shard(code) for code in placements
|
|
]
|
|
|
|
|
|
def publish_destination_placements(
|
|
comm: Any,
|
|
device: torch.device,
|
|
stream: torch.cuda.Stream,
|
|
first_worker_rank: int,
|
|
placements: "Sequence[Placements | None] | None",
|
|
num_parameters: int,
|
|
) -> list[Placements | None]:
|
|
"""Share the worker-side destination plan with every rank in the group.
|
|
|
|
The trainer must issue each reshard with the same destination the workers
|
|
use, but once destinations are per-parameter they depend on the inference
|
|
model, which only the workers can see. The first worker publishes them here,
|
|
over the shared NCCL communicator; trainer ranks pass `None` and receive
|
|
them, and the other workers pass their own so a disagreement is caught
|
|
rather than deadlocking later.
|
|
|
|
Only placements travel: the destination *mesh* is still derived from the
|
|
rank split, identically on both sides. The plan is published on the data
|
|
plane before any M2N reshard collective.
|
|
"""
|
|
replicated_sentinel = REPLICATE - 1
|
|
encoded = torch.full(
|
|
(num_parameters, MESH_NDIMS),
|
|
replicated_sentinel,
|
|
dtype=torch.int8,
|
|
device=device,
|
|
)
|
|
if comm.rank == first_worker_rank:
|
|
if placements is None or len(placements) != num_parameters:
|
|
raise ValueError(
|
|
f"publishing rank needs {num_parameters} destination placements"
|
|
)
|
|
rows = [
|
|
[replicated_sentinel] * MESH_NDIMS if placement is REPLICATED else placement
|
|
for placement in placements
|
|
]
|
|
if rows:
|
|
encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=device))
|
|
|
|
comm.broadcast(encoded, encoded, root=first_worker_rank, stream=stream)
|
|
return [
|
|
REPLICATED
|
|
if all(code == replicated_sentinel for code in row)
|
|
else cast(Placements, tuple(row))
|
|
for row in encoded.tolist()
|
|
]
|