Files
vLLM/vllm/distributed/weight_transfer/m2n_trainer.py
T
6b6a6c5c98 [RL] Support per-parameter source layouts for NCCL M2N (#60934)
Signed-off-by: pkousha <pkousha@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
2026-10-16 21:41:17 -04:00

629 lines
25 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Trainer-side weight transfer engine for the NCCL M2N backend.
Symmetric to `M2NWeightTransferEngine` but in the training process. Every
trainer rank joins the shared communicator and runs every reshard, sending its
own local shard; only rank 0 touches the inference control plane.
Workers join the communicator during init and run reshard from inside
`update_weights`, so those two RPCs overlap with work on this side. Both run on
a helper thread and are joined afterwards, which keeps that overlap inside the
engine -- where the `TrainerWeightTransferEngine` contract puts it -- rather than
in every caller.
"""
import socket
from collections.abc import Mapping
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar, Self, cast
import torch
from vllm.distributed.weight_transfer.base import (
TrainerInitInfo,
TrainerWeightTransferEngine,
VLLMWeightSyncClient,
WeightSource,
)
from vllm.distributed.weight_transfer.m2n_common import (
M2N_WIRE_SCHEMA_VERSION,
M2NMesh,
M2NParamMeta,
M2NWireParam,
Placements,
check_complete_plan_agreement,
check_data_plane_agreement,
check_runtime_ready_agreement,
check_source_plan_agreement,
check_transferable,
create_m2n_communicator,
expected_local_shape,
import_m2n,
prepare_m2n_local_runtime,
publish_destination_placements,
resolve_layout,
source_plan_digest,
to_mesh,
to_placements,
validate_local_buffer,
validate_transfer_plan,
)
from vllm.distributed.weight_transfer.m2n_source import M2NWeightSource
from vllm.distributed.weight_transfer.nccl_common import (
decode_nccl_unique_id,
stateless_init_metadata_group,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.distributed.utils import StatelessProcessGroup
logger = init_logger(__name__)
__all__ = ["M2NTrainerInitInfo", "M2NTrainerWeightTransferEngine"]
@dataclass(kw_only=True)
class M2NTrainerInitInfo(TrainerInitInfo):
"""Trainer-side init info for nccl_m2n.
`rank` (from `TrainerInitInfo`) is this trainer process's rank; it is also
its rank in the shared communicator, since the trainer occupies
`[0, num_trainer_ranks)`. Rank 0 drives the control plane.
"""
backend: ClassVar[str] = "nccl_m2n"
master_address: str
master_port: int
world_size: int
"""Trainer ranks + all inference workers."""
num_trainer_ranks: int = 1
dst_mesh_dims: tuple[int, int] | None = None
"""How the inference ranks are laid out: axis 0 replicates and axis 1
shards. The trainer declares it so both sides describe the destination
identically; defaults to a flat `(num_workers, 1)`, which is all a
replicated destination needs.
Destination meshes remain common to every parameter; per-parameter
destination meshes are intentionally future work.
"""
nccl_unique_id_b64: str | None = field(default=None, repr=False)
"""Optional UID for the NCCL data plane; TCP remains the metadata plane."""
max_cta: int | None = None
listen_socket: socket.socket | None = field(default=None, repr=False, compare=False)
"""Optional rank-zero TCPStore reservation; never sent to workers."""
@property
def destination_mesh_dims(self) -> tuple[int, int]:
"""`dst_mesh_dims`, or a flat mesh over every inference worker."""
if self.dst_mesh_dims is not None:
return cast(tuple[int, int], tuple(self.dst_mesh_dims))
return (self.world_size - self.num_trainer_ranks, 1)
def __post_init__(self) -> None:
"""Reject rank counts or destination meshes that cannot form a group."""
if not isinstance(self.master_address, str) or not self.master_address:
raise ValueError("`master_address` must name the metadata rendezvous")
if (
not isinstance(self.master_port, int)
or isinstance(self.master_port, bool)
or not 0 < self.master_port < 65536
):
raise ValueError("`master_port` must be a valid metadata port")
if not 0 < self.num_trainer_ranks < self.world_size:
raise ValueError(
f"`num_trainer_ranks` ({self.num_trainer_ranks}) must leave at "
f"least one worker in world_size {self.world_size}"
)
num_workers = self.world_size - self.num_trainer_ranks
if self.dst_mesh_dims is not None:
dims = tuple(self.dst_mesh_dims)
try:
mesh = M2NMesh(cast(tuple[int, int], dims), self.num_trainer_ranks)
except (TypeError, ValueError) as exc:
raise ValueError(f"invalid `dst_mesh_dims` {dims}: {exc}") from exc
if mesh.size != num_workers:
raise ValueError(
f"`dst_mesh_dims` {dims} must be two dims covering the "
f"{num_workers} inference workers"
)
if not 0 <= self.rank < self.num_trainer_ranks:
raise ValueError(
f"trainer `rank` ({self.rank}) must be below "
f"`num_trainer_ranks` ({self.num_trainer_ranks})"
)
if self.listen_socket is not None:
if self.rank != 0:
raise ValueError("only trainer rank zero may reserve the TCPStore")
if not isinstance(self.listen_socket, socket.socket):
raise TypeError("listen_socket must be a socket.socket")
if self.listen_socket.fileno() < 0:
raise ValueError("listen_socket is already closed")
@property
def nccl_unique_id_bytes(self) -> bytes | None:
if self.nccl_unique_id_b64 is None:
return None
return decode_nccl_unique_id(
master_address=None,
master_port=None,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="M2NTrainerInitInfo data plane",
)
class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitInfo]):
"""Trainer-side engine: sends every rank's local shard, once per parameter.
Called on every trainer rank. All ranks join the communicator and run every
reshard; only rank 0 touches the control plane.
"""
init_info_cls = M2NTrainerInitInfo
def __init__(
self,
*,
client: VLLMWeightSyncClient,
source: M2NWeightSource,
is_sender: bool,
) -> None:
"""Hold the client and source; `trainer_init` fills in the group."""
super().__init__(client=client, source=source, is_sender=is_sender)
self.group: Any = None
self._m2n: Any = None
self._handle: Any = None
self._metas: list[M2NParamMeta] = []
self._wire_params: list[M2NWireParam] = []
self._source_digest = ""
self._dst_mesh: M2NMesh | None = None
self._dst_placements: list[Placements | None] = []
self._metadata_group: StatelessProcessGroup | None = None
self._num_trainer_ranks = 0
@classmethod
def trainer_init(
cls,
init_info: M2NTrainerInitInfo,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
) -> Self:
"""Build the engine and rendezvous with the inference side.
Runs on *every* trainer rank. Rank 0 additionally ships the transfer
plan to the workers and drives the control plane; the other ranks only
build local state and join the communicator. Every rank participates
in every reshard and sends its local shard.
Build a local validation envelope, start the worker RPC without
waiting, join the metadata rendezvous, and require unanimous source,
data-plane, and runtime readiness before constructing NCCL.
"""
engine = cls(
client=client,
source=cast(M2NWeightSource, source),
is_sender=init_info.is_sender,
)
engine._initialize(init_info, source)
return engine
def _initialize(
self,
init_info: M2NTrainerInitInfo,
source: WeightSource | None,
) -> None:
"""Run source validation, metadata agreement, and NCCL rendezvous."""
self._num_trainer_ranks = init_info.num_trainer_ranks
local_source_error: str | None = None
try:
if not isinstance(source, M2NWeightSource):
raise TypeError(
"nccl_m2n needs per-parameter layouts, so its source must "
f"be an M2NWeightSource; got {type(source).__name__}"
)
self._m2n = import_m2n()
self._prepare_source_plan(source, init_info.num_trainer_ranks)
except Exception as exc:
local_source_error = f"{type(exc).__name__}: {exc}"
self._metas = []
self._wire_params = []
self._source_digest = source_plan_digest([])
executor: ThreadPoolExecutor | None = None
init_future: Future[None] | None = None
if self.is_sender:
executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="m2n-weight-sync"
)
init_future = executor.submit(
self.client.init_weight_transfer_engine,
self._worker_init_info(init_info),
)
handle = None
group = None
try:
reservation = init_info.listen_socket
try:
metadata_group = stateless_init_metadata_group(
init_info.master_address,
init_info.master_port,
init_info.rank,
init_info.world_size,
listen_socket=reservation,
)
finally:
init_info.listen_socket = None
if reservation is not None:
reservation.close()
plan_digest = check_source_plan_agreement(
metadata_group,
self._wire_params,
expected_digest=self._source_digest,
local_error=local_source_error,
)
# Retain the common destination mesh and the merged NCCL publication
# of per-parameter placements.
self._dst_mesh = M2NMesh(
init_info.destination_mesh_dims, init_info.num_trainer_ranks
)
unique_id_bytes: bytes | None = None
data_plane_error: str | None = None
try:
unique_id_bytes = init_info.nccl_unique_id_bytes
except Exception as exc:
data_plane_error = f"{type(exc).__name__}: {exc}"
check_data_plane_agreement(
metadata_group,
unique_id_bytes,
data_plane_error,
)
runtime_error: str | None = None
nccl = None
device = None
try:
nccl, device, handle = prepare_m2n_local_runtime(
self._m2n, init_info.max_cta
)
except Exception as exc:
runtime_error = f"{type(exc).__name__}: {exc}"
check_runtime_ready_agreement(metadata_group, runtime_error)
assert nccl is not None and device is not None and handle is not None
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
destination_placements = self._receive_destination_plan(
group,
init_info.num_trainer_ranks,
torch.device("cuda", device),
)
complete_plan_error: str | None = None
try:
assert self._dst_mesh is not None
validate_transfer_plan(
self._metas, self._dst_mesh, destination_placements
)
except Exception as exc:
complete_plan_error = f"{type(exc).__name__}: {exc}"
assert self._dst_mesh is not None
check_complete_plan_agreement(
metadata_group,
plan_digest,
self._dst_mesh,
destination_placements,
complete_plan_error,
)
if init_future is not None:
init_future.result()
logger.info(
"nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), "
"inference ranks [%d, %d)",
len(self._metas),
init_info.num_trainer_ranks,
init_info.num_trainer_ranks,
init_info.world_size,
)
logger.info("nccl_m2n source plan digest: %s", plan_digest)
except BaseException:
if group is not None:
try:
# Transport cleanup only; reload validity belongs to the
# model-update lifecycle, not this backend.
group.abort()
except BaseException:
logger.exception(
"failed to abort partially initialized nccl_m2n communicator"
)
if handle is not None:
try:
handle.destroy()
except BaseException:
logger.exception(
"failed to destroy partially initialized nccl_m2n handle"
)
raise
finally:
if executor is not None:
executor.shutdown(wait=False, cancel_futures=True)
self._metadata_group = metadata_group
self._handle = handle
self.group = group
self._dst_placements = destination_placements
def _receive_destination_plan(
self, group: Any, first_worker_rank: int, device: torch.device
) -> list[Placements | None]:
"""Receive the inference-side per-parameter placements."""
# The first inference worker publishes one placement per parameter.
# Every trainer rank receives the complete plan in this single,
# initialization-time broadcast.
placements = publish_destination_placements(
group,
device,
torch.cuda.current_stream(),
first_worker_rank,
None,
len(self._metas),
)
return placements
def _worker_init_info(self, init_info: M2NTrainerInitInfo) -> dict[str, Any]:
"""Handshake payload: rendezvous, both meshes, and the transfer plan."""
payload: dict[str, Any] = {
"schema_version": M2N_WIRE_SCHEMA_VERSION,
"master_address": init_info.master_address,
"master_port": init_info.master_port,
"nccl_unique_id_b64": init_info.nccl_unique_id_b64,
"rank_offset": init_info.num_trainer_ranks,
"world_size": init_info.world_size,
"dst_mesh_dims": list(init_info.destination_mesh_dims),
"source_digest": self._source_digest,
"params": [param.to_dict() for param in self._wire_params],
"max_cta": init_info.max_cta,
}
return {key: value for key, value in payload.items() if value is not None}
def _prepare_source_plan(
self, source: M2NWeightSource, num_trainer_ranks: int
) -> None:
"""Build and validate this rank's immutable source plan."""
self._metas = source.metadata()
names: set[str] = set()
for meta in self._metas:
if not isinstance(meta, M2NParamMeta):
raise TypeError(
"nccl_m2n source metadata must contain M2NParamMeta "
f"entries, got {type(meta).__name__}"
)
if meta.name in names:
raise ValueError(
"nccl_m2n source metadata contains duplicate parameter "
f"'{meta.name}'"
)
names.add(meta.name)
check_transferable(meta.name, meta.dtype, meta.shape)
layout = meta.source_layout
if layout.mesh.start_rank != 0:
raise ValueError(
f"parameter '{meta.name}' source mesh must start at rank 0, "
f"got {layout.mesh.start_rank}"
)
if layout.mesh.size != num_trainer_ranks:
raise ValueError(
f"parameter '{meta.name}' source mesh covers "
f"{layout.mesh.size} ranks, but there are "
f"{num_trainer_ranks} trainer ranks"
)
expected_local_shape(meta)
self._wire_params = [
M2NWireParam(
name=meta.name,
dtype_name=str(meta.dtype).split(".")[-1],
shape=meta.shape,
src_mesh_dims=meta.source_layout.mesh.dims,
src_placements=meta.source_layout.placements,
)
for meta in self._metas
]
self._source_digest = source_plan_digest(self._wire_params)
def send_weights(self) -> None:
"""Drive one update round: start, reshard concurrently, then finish."""
if self._handle is None or self.group is None:
raise RuntimeError("nccl_m2n trainer engine is not initialized")
if not self.is_sender:
try:
self._send()
except BaseException:
try:
self._receive_round_outcome()
except BaseException:
logger.exception(
"failed to consume nccl_m2n round outcome after local failure"
)
raise
self._receive_round_outcome()
return
try:
self.client.start_weight_update()
executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="m2n-weight-sync"
)
try:
update = executor.submit(
self.client.update_weights,
{"names": [m.name for m in self._metas]},
)
# Surface an immediate worker-side rejection before trainers
# enter the collective sequence.
if update.done():
update.result()
self._send()
update.result()
finally:
executor.shutdown(wait=False, cancel_futures=True)
self.client.finish_weight_update()
except BaseException as exc:
try:
self._publish_round_outcome(exc)
except BaseException:
logger.exception("failed to publish nccl_m2n round failure")
raise
self._publish_round_outcome(None)
def _publish_round_outcome(self, failure: BaseException | None) -> None:
"""Tell non-sender trainer ranks whether worker finalization passed."""
if self._num_trainer_ranks <= 1:
return
if self._metadata_group is None:
raise RuntimeError("nccl_m2n metadata group is unavailable")
envelope = {
"ok": failure is None,
"error": (
None if failure is None else f"{type(failure).__name__}: {failure}"
),
}
for rank in range(1, self._num_trainer_ranks):
self._metadata_group.send_obj(envelope, rank)
# Keep rank zero's TCPStore alive until every trainer consumed its
# outcome. This also keeps the per-rank message counters aligned.
expected_ack = {"phase": "round_outcome_consumed"}
for rank in range(1, self._num_trainer_ranks):
ack = self._metadata_group.recv_obj(rank)
if ack != expected_ack:
raise RuntimeError(
f"trainer rank {rank} sent an invalid round-outcome acknowledgement"
)
def _receive_round_outcome(self) -> None:
"""Wait until trainer rank zero observed every worker's finish RPC."""
if self._metadata_group is None:
raise RuntimeError("nccl_m2n metadata group is unavailable")
outcome = self._metadata_group.recv_obj(0)
self._metadata_group.send_obj({"phase": "round_outcome_consumed"}, 0)
if not isinstance(outcome, Mapping) or set(outcome) != {"ok", "error"}:
raise RuntimeError("trainer rank zero sent an invalid round outcome")
ok = outcome["ok"]
error = outcome["error"]
if not isinstance(ok, bool) or ok != (error is None):
raise RuntimeError("trainer rank zero sent an invalid round outcome")
if not ok:
if not isinstance(error, str) or not error:
raise RuntimeError("trainer rank zero sent an invalid round error")
raise RuntimeError(
"nccl_m2n worker update failed on trainer rank zero: " + error
)
def _send(self) -> None:
"""Reshard each local shard into its worker-planned destination."""
assert self.group is not None
assert self._dst_mesh is not None
assert self.source is not None
m2n = self._m2n
stream = torch.cuda.current_stream()
device_index = torch.accelerator.current_device_index()
# Optimizer work may have used another CUDA stream. Make prior writes
# visible before a provider materializes current-stream local shards.
torch.accelerator.synchronize(device_index)
inflight_buffers: list[Any] = []
sent = 0
for index, (name, buffer) in enumerate(self.source):
if index >= len(self._metas):
raise RuntimeError(
"weight source yielded more parameters than metadata() "
f"declared (first extra parameter: '{name}')"
)
meta = self._metas[index]
if name != meta.name:
raise RuntimeError(
f"weight source yielded '{name}' at position {index}, but "
f"its metadata declared '{meta.name}'; the source must "
"iterate in a stable order"
)
validate_local_buffer(m2n, meta, buffer)
src_mesh, src_placements = resolve_layout(
meta.source_layout.mesh, meta.source_layout.placements
)
dst_mesh, dst_placements = resolve_layout(
self._dst_mesh, self._dst_placements[index]
)
raw_pointer_args = {}
if isinstance(buffer, int):
raw_pointer_args = {
"src_local_shape": expected_local_shape(meta),
"src_dtype": meta.dtype,
}
m2n.reshard(
buffer,
None,
self.group,
stream,
src_mesh=to_mesh(m2n, src_mesh),
src_placements=to_placements(m2n, src_placements),
dst_mesh=to_mesh(m2n, dst_mesh),
dst_placements=to_placements(m2n, dst_placements),
handle=self._handle,
**raw_pointer_args,
)
# Let allocator-aware buffers recycle storage in stream order.
# Other object owners must stay alive until the final synchronize;
# raw-pointer lifetime remains the source's responsibility.
record_stream = getattr(buffer, "record_stream", None)
if callable(record_stream):
record_stream(stream)
elif not isinstance(buffer, int):
inflight_buffers.append(buffer)
sent += 1
if sent != len(self._metas):
missing = self._metas[sent].name
raise RuntimeError(
f"weight source yielded {sent} parameters, but metadata() "
f"declared {len(self._metas)} (first missing: '{missing}')"
)
# As in the other weight-transfer backends, wait before returning
# rather than relying on same-stream ordering. The caller may then
# safely mutate parameters or continue work on another stream.
stream.synchronize()
def shutdown(self) -> None:
"""Finish pending GPU work and release M2N communication state."""
handle, self._handle = self._handle, None
group, self.group = self.group, None
self._metadata_group = None
self._dst_placements = []
self._dst_mesh = None
self._m2n = None
first_error: BaseException | None = None
def attempt(action: Any, message: str) -> None:
nonlocal first_error
try:
action()
except BaseException as exc:
if first_error is None:
first_error = exc
logger.exception(message)
if handle is not None or group is not None:
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
if handle is not None:
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
if group is not None:
attempt(group.abort, "failed to abort nccl_m2n communicator")
if first_error is not None:
raise first_error