Signed-off-by: pkousha <pkousha@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
629 lines
25 KiB
Python
629 lines
25 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Trainer-side weight transfer engine for the NCCL M2N backend.
|
|
|
|
Symmetric to `M2NWeightTransferEngine` but in the training process. Every
|
|
trainer rank joins the shared communicator and runs every reshard, sending its
|
|
own local shard; only rank 0 touches the inference control plane.
|
|
|
|
Workers join the communicator during init and run reshard from inside
|
|
`update_weights`, so those two RPCs overlap with work on this side. Both run on
|
|
a helper thread and are joined afterwards, which keeps that overlap inside the
|
|
engine -- where the `TrainerWeightTransferEngine` contract puts it -- rather than
|
|
in every caller.
|
|
"""
|
|
|
|
import socket
|
|
from collections.abc import Mapping
|
|
from concurrent.futures import Future, ThreadPoolExecutor
|
|
from dataclasses import dataclass, field
|
|
from typing import TYPE_CHECKING, Any, ClassVar, Self, cast
|
|
|
|
import torch
|
|
|
|
from vllm.distributed.weight_transfer.base import (
|
|
TrainerInitInfo,
|
|
TrainerWeightTransferEngine,
|
|
VLLMWeightSyncClient,
|
|
WeightSource,
|
|
)
|
|
from vllm.distributed.weight_transfer.m2n_common import (
|
|
M2N_WIRE_SCHEMA_VERSION,
|
|
M2NMesh,
|
|
M2NParamMeta,
|
|
M2NWireParam,
|
|
Placements,
|
|
check_complete_plan_agreement,
|
|
check_data_plane_agreement,
|
|
check_runtime_ready_agreement,
|
|
check_source_plan_agreement,
|
|
check_transferable,
|
|
create_m2n_communicator,
|
|
expected_local_shape,
|
|
import_m2n,
|
|
prepare_m2n_local_runtime,
|
|
publish_destination_placements,
|
|
resolve_layout,
|
|
source_plan_digest,
|
|
to_mesh,
|
|
to_placements,
|
|
validate_local_buffer,
|
|
validate_transfer_plan,
|
|
)
|
|
from vllm.distributed.weight_transfer.m2n_source import M2NWeightSource
|
|
from vllm.distributed.weight_transfer.nccl_common import (
|
|
decode_nccl_unique_id,
|
|
stateless_init_metadata_group,
|
|
)
|
|
from vllm.logger import init_logger
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.distributed.utils import StatelessProcessGroup
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
__all__ = ["M2NTrainerInitInfo", "M2NTrainerWeightTransferEngine"]
|
|
|
|
|
|
@dataclass(kw_only=True)
|
|
class M2NTrainerInitInfo(TrainerInitInfo):
|
|
"""Trainer-side init info for nccl_m2n.
|
|
|
|
`rank` (from `TrainerInitInfo`) is this trainer process's rank; it is also
|
|
its rank in the shared communicator, since the trainer occupies
|
|
`[0, num_trainer_ranks)`. Rank 0 drives the control plane.
|
|
"""
|
|
|
|
backend: ClassVar[str] = "nccl_m2n"
|
|
|
|
master_address: str
|
|
master_port: int
|
|
world_size: int
|
|
"""Trainer ranks + all inference workers."""
|
|
num_trainer_ranks: int = 1
|
|
dst_mesh_dims: tuple[int, int] | None = None
|
|
"""How the inference ranks are laid out: axis 0 replicates and axis 1
|
|
shards. The trainer declares it so both sides describe the destination
|
|
identically; defaults to a flat `(num_workers, 1)`, which is all a
|
|
replicated destination needs.
|
|
|
|
Destination meshes remain common to every parameter; per-parameter
|
|
destination meshes are intentionally future work.
|
|
"""
|
|
nccl_unique_id_b64: str | None = field(default=None, repr=False)
|
|
"""Optional UID for the NCCL data plane; TCP remains the metadata plane."""
|
|
max_cta: int | None = None
|
|
listen_socket: socket.socket | None = field(default=None, repr=False, compare=False)
|
|
"""Optional rank-zero TCPStore reservation; never sent to workers."""
|
|
|
|
@property
|
|
def destination_mesh_dims(self) -> tuple[int, int]:
|
|
"""`dst_mesh_dims`, or a flat mesh over every inference worker."""
|
|
if self.dst_mesh_dims is not None:
|
|
return cast(tuple[int, int], tuple(self.dst_mesh_dims))
|
|
return (self.world_size - self.num_trainer_ranks, 1)
|
|
|
|
def __post_init__(self) -> None:
|
|
"""Reject rank counts or destination meshes that cannot form a group."""
|
|
if not isinstance(self.master_address, str) or not self.master_address:
|
|
raise ValueError("`master_address` must name the metadata rendezvous")
|
|
if (
|
|
not isinstance(self.master_port, int)
|
|
or isinstance(self.master_port, bool)
|
|
or not 0 < self.master_port < 65536
|
|
):
|
|
raise ValueError("`master_port` must be a valid metadata port")
|
|
if not 0 < self.num_trainer_ranks < self.world_size:
|
|
raise ValueError(
|
|
f"`num_trainer_ranks` ({self.num_trainer_ranks}) must leave at "
|
|
f"least one worker in world_size {self.world_size}"
|
|
)
|
|
num_workers = self.world_size - self.num_trainer_ranks
|
|
if self.dst_mesh_dims is not None:
|
|
dims = tuple(self.dst_mesh_dims)
|
|
try:
|
|
mesh = M2NMesh(cast(tuple[int, int], dims), self.num_trainer_ranks)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError(f"invalid `dst_mesh_dims` {dims}: {exc}") from exc
|
|
if mesh.size != num_workers:
|
|
raise ValueError(
|
|
f"`dst_mesh_dims` {dims} must be two dims covering the "
|
|
f"{num_workers} inference workers"
|
|
)
|
|
if not 0 <= self.rank < self.num_trainer_ranks:
|
|
raise ValueError(
|
|
f"trainer `rank` ({self.rank}) must be below "
|
|
f"`num_trainer_ranks` ({self.num_trainer_ranks})"
|
|
)
|
|
if self.listen_socket is not None:
|
|
if self.rank != 0:
|
|
raise ValueError("only trainer rank zero may reserve the TCPStore")
|
|
if not isinstance(self.listen_socket, socket.socket):
|
|
raise TypeError("listen_socket must be a socket.socket")
|
|
if self.listen_socket.fileno() < 0:
|
|
raise ValueError("listen_socket is already closed")
|
|
|
|
@property
|
|
def nccl_unique_id_bytes(self) -> bytes | None:
|
|
if self.nccl_unique_id_b64 is None:
|
|
return None
|
|
return decode_nccl_unique_id(
|
|
master_address=None,
|
|
master_port=None,
|
|
nccl_unique_id_b64=self.nccl_unique_id_b64,
|
|
ctx="M2NTrainerInitInfo data plane",
|
|
)
|
|
|
|
|
|
class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitInfo]):
|
|
"""Trainer-side engine: sends every rank's local shard, once per parameter.
|
|
|
|
Called on every trainer rank. All ranks join the communicator and run every
|
|
reshard; only rank 0 touches the control plane.
|
|
"""
|
|
|
|
init_info_cls = M2NTrainerInitInfo
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
client: VLLMWeightSyncClient,
|
|
source: M2NWeightSource,
|
|
is_sender: bool,
|
|
) -> None:
|
|
"""Hold the client and source; `trainer_init` fills in the group."""
|
|
super().__init__(client=client, source=source, is_sender=is_sender)
|
|
self.group: Any = None
|
|
self._m2n: Any = None
|
|
self._handle: Any = None
|
|
self._metas: list[M2NParamMeta] = []
|
|
self._wire_params: list[M2NWireParam] = []
|
|
self._source_digest = ""
|
|
self._dst_mesh: M2NMesh | None = None
|
|
self._dst_placements: list[Placements | None] = []
|
|
self._metadata_group: StatelessProcessGroup | None = None
|
|
self._num_trainer_ranks = 0
|
|
|
|
@classmethod
|
|
def trainer_init(
|
|
cls,
|
|
init_info: M2NTrainerInitInfo,
|
|
*,
|
|
client: VLLMWeightSyncClient,
|
|
source: WeightSource | None = None,
|
|
) -> Self:
|
|
"""Build the engine and rendezvous with the inference side.
|
|
|
|
Runs on *every* trainer rank. Rank 0 additionally ships the transfer
|
|
plan to the workers and drives the control plane; the other ranks only
|
|
build local state and join the communicator. Every rank participates
|
|
in every reshard and sends its local shard.
|
|
|
|
Build a local validation envelope, start the worker RPC without
|
|
waiting, join the metadata rendezvous, and require unanimous source,
|
|
data-plane, and runtime readiness before constructing NCCL.
|
|
"""
|
|
engine = cls(
|
|
client=client,
|
|
source=cast(M2NWeightSource, source),
|
|
is_sender=init_info.is_sender,
|
|
)
|
|
engine._initialize(init_info, source)
|
|
return engine
|
|
|
|
def _initialize(
|
|
self,
|
|
init_info: M2NTrainerInitInfo,
|
|
source: WeightSource | None,
|
|
) -> None:
|
|
"""Run source validation, metadata agreement, and NCCL rendezvous."""
|
|
self._num_trainer_ranks = init_info.num_trainer_ranks
|
|
local_source_error: str | None = None
|
|
try:
|
|
if not isinstance(source, M2NWeightSource):
|
|
raise TypeError(
|
|
"nccl_m2n needs per-parameter layouts, so its source must "
|
|
f"be an M2NWeightSource; got {type(source).__name__}"
|
|
)
|
|
self._m2n = import_m2n()
|
|
self._prepare_source_plan(source, init_info.num_trainer_ranks)
|
|
except Exception as exc:
|
|
local_source_error = f"{type(exc).__name__}: {exc}"
|
|
self._metas = []
|
|
self._wire_params = []
|
|
self._source_digest = source_plan_digest([])
|
|
|
|
executor: ThreadPoolExecutor | None = None
|
|
init_future: Future[None] | None = None
|
|
if self.is_sender:
|
|
executor = ThreadPoolExecutor(
|
|
max_workers=1, thread_name_prefix="m2n-weight-sync"
|
|
)
|
|
init_future = executor.submit(
|
|
self.client.init_weight_transfer_engine,
|
|
self._worker_init_info(init_info),
|
|
)
|
|
handle = None
|
|
group = None
|
|
try:
|
|
reservation = init_info.listen_socket
|
|
try:
|
|
metadata_group = stateless_init_metadata_group(
|
|
init_info.master_address,
|
|
init_info.master_port,
|
|
init_info.rank,
|
|
init_info.world_size,
|
|
listen_socket=reservation,
|
|
)
|
|
finally:
|
|
init_info.listen_socket = None
|
|
if reservation is not None:
|
|
reservation.close()
|
|
plan_digest = check_source_plan_agreement(
|
|
metadata_group,
|
|
self._wire_params,
|
|
expected_digest=self._source_digest,
|
|
local_error=local_source_error,
|
|
)
|
|
|
|
# Retain the common destination mesh and the merged NCCL publication
|
|
# of per-parameter placements.
|
|
self._dst_mesh = M2NMesh(
|
|
init_info.destination_mesh_dims, init_info.num_trainer_ranks
|
|
)
|
|
unique_id_bytes: bytes | None = None
|
|
data_plane_error: str | None = None
|
|
try:
|
|
unique_id_bytes = init_info.nccl_unique_id_bytes
|
|
except Exception as exc:
|
|
data_plane_error = f"{type(exc).__name__}: {exc}"
|
|
check_data_plane_agreement(
|
|
metadata_group,
|
|
unique_id_bytes,
|
|
data_plane_error,
|
|
)
|
|
|
|
runtime_error: str | None = None
|
|
nccl = None
|
|
device = None
|
|
try:
|
|
nccl, device, handle = prepare_m2n_local_runtime(
|
|
self._m2n, init_info.max_cta
|
|
)
|
|
except Exception as exc:
|
|
runtime_error = f"{type(exc).__name__}: {exc}"
|
|
check_runtime_ready_agreement(metadata_group, runtime_error)
|
|
assert nccl is not None and device is not None and handle is not None
|
|
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
|
|
destination_placements = self._receive_destination_plan(
|
|
group,
|
|
init_info.num_trainer_ranks,
|
|
torch.device("cuda", device),
|
|
)
|
|
|
|
complete_plan_error: str | None = None
|
|
try:
|
|
assert self._dst_mesh is not None
|
|
validate_transfer_plan(
|
|
self._metas, self._dst_mesh, destination_placements
|
|
)
|
|
except Exception as exc:
|
|
complete_plan_error = f"{type(exc).__name__}: {exc}"
|
|
assert self._dst_mesh is not None
|
|
check_complete_plan_agreement(
|
|
metadata_group,
|
|
plan_digest,
|
|
self._dst_mesh,
|
|
destination_placements,
|
|
complete_plan_error,
|
|
)
|
|
|
|
if init_future is not None:
|
|
init_future.result()
|
|
|
|
logger.info(
|
|
"nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), "
|
|
"inference ranks [%d, %d)",
|
|
len(self._metas),
|
|
init_info.num_trainer_ranks,
|
|
init_info.num_trainer_ranks,
|
|
init_info.world_size,
|
|
)
|
|
logger.info("nccl_m2n source plan digest: %s", plan_digest)
|
|
except BaseException:
|
|
if group is not None:
|
|
try:
|
|
# Transport cleanup only; reload validity belongs to the
|
|
# model-update lifecycle, not this backend.
|
|
group.abort()
|
|
except BaseException:
|
|
logger.exception(
|
|
"failed to abort partially initialized nccl_m2n communicator"
|
|
)
|
|
if handle is not None:
|
|
try:
|
|
handle.destroy()
|
|
except BaseException:
|
|
logger.exception(
|
|
"failed to destroy partially initialized nccl_m2n handle"
|
|
)
|
|
raise
|
|
finally:
|
|
if executor is not None:
|
|
executor.shutdown(wait=False, cancel_futures=True)
|
|
|
|
self._metadata_group = metadata_group
|
|
self._handle = handle
|
|
self.group = group
|
|
self._dst_placements = destination_placements
|
|
|
|
def _receive_destination_plan(
|
|
self, group: Any, first_worker_rank: int, device: torch.device
|
|
) -> list[Placements | None]:
|
|
"""Receive the inference-side per-parameter placements."""
|
|
# The first inference worker publishes one placement per parameter.
|
|
# Every trainer rank receives the complete plan in this single,
|
|
# initialization-time broadcast.
|
|
placements = publish_destination_placements(
|
|
group,
|
|
device,
|
|
torch.cuda.current_stream(),
|
|
first_worker_rank,
|
|
None,
|
|
len(self._metas),
|
|
)
|
|
return placements
|
|
|
|
def _worker_init_info(self, init_info: M2NTrainerInitInfo) -> dict[str, Any]:
|
|
"""Handshake payload: rendezvous, both meshes, and the transfer plan."""
|
|
payload: dict[str, Any] = {
|
|
"schema_version": M2N_WIRE_SCHEMA_VERSION,
|
|
"master_address": init_info.master_address,
|
|
"master_port": init_info.master_port,
|
|
"nccl_unique_id_b64": init_info.nccl_unique_id_b64,
|
|
"rank_offset": init_info.num_trainer_ranks,
|
|
"world_size": init_info.world_size,
|
|
"dst_mesh_dims": list(init_info.destination_mesh_dims),
|
|
"source_digest": self._source_digest,
|
|
"params": [param.to_dict() for param in self._wire_params],
|
|
"max_cta": init_info.max_cta,
|
|
}
|
|
return {key: value for key, value in payload.items() if value is not None}
|
|
|
|
def _prepare_source_plan(
|
|
self, source: M2NWeightSource, num_trainer_ranks: int
|
|
) -> None:
|
|
"""Build and validate this rank's immutable source plan."""
|
|
self._metas = source.metadata()
|
|
names: set[str] = set()
|
|
for meta in self._metas:
|
|
if not isinstance(meta, M2NParamMeta):
|
|
raise TypeError(
|
|
"nccl_m2n source metadata must contain M2NParamMeta "
|
|
f"entries, got {type(meta).__name__}"
|
|
)
|
|
if meta.name in names:
|
|
raise ValueError(
|
|
"nccl_m2n source metadata contains duplicate parameter "
|
|
f"'{meta.name}'"
|
|
)
|
|
names.add(meta.name)
|
|
check_transferable(meta.name, meta.dtype, meta.shape)
|
|
layout = meta.source_layout
|
|
if layout.mesh.start_rank != 0:
|
|
raise ValueError(
|
|
f"parameter '{meta.name}' source mesh must start at rank 0, "
|
|
f"got {layout.mesh.start_rank}"
|
|
)
|
|
if layout.mesh.size != num_trainer_ranks:
|
|
raise ValueError(
|
|
f"parameter '{meta.name}' source mesh covers "
|
|
f"{layout.mesh.size} ranks, but there are "
|
|
f"{num_trainer_ranks} trainer ranks"
|
|
)
|
|
expected_local_shape(meta)
|
|
self._wire_params = [
|
|
M2NWireParam(
|
|
name=meta.name,
|
|
dtype_name=str(meta.dtype).split(".")[-1],
|
|
shape=meta.shape,
|
|
src_mesh_dims=meta.source_layout.mesh.dims,
|
|
src_placements=meta.source_layout.placements,
|
|
)
|
|
for meta in self._metas
|
|
]
|
|
self._source_digest = source_plan_digest(self._wire_params)
|
|
|
|
def send_weights(self) -> None:
|
|
"""Drive one update round: start, reshard concurrently, then finish."""
|
|
if self._handle is None or self.group is None:
|
|
raise RuntimeError("nccl_m2n trainer engine is not initialized")
|
|
|
|
if not self.is_sender:
|
|
try:
|
|
self._send()
|
|
except BaseException:
|
|
try:
|
|
self._receive_round_outcome()
|
|
except BaseException:
|
|
logger.exception(
|
|
"failed to consume nccl_m2n round outcome after local failure"
|
|
)
|
|
raise
|
|
self._receive_round_outcome()
|
|
return
|
|
|
|
try:
|
|
self.client.start_weight_update()
|
|
executor = ThreadPoolExecutor(
|
|
max_workers=1, thread_name_prefix="m2n-weight-sync"
|
|
)
|
|
try:
|
|
update = executor.submit(
|
|
self.client.update_weights,
|
|
{"names": [m.name for m in self._metas]},
|
|
)
|
|
# Surface an immediate worker-side rejection before trainers
|
|
# enter the collective sequence.
|
|
if update.done():
|
|
update.result()
|
|
self._send()
|
|
update.result()
|
|
finally:
|
|
executor.shutdown(wait=False, cancel_futures=True)
|
|
self.client.finish_weight_update()
|
|
except BaseException as exc:
|
|
try:
|
|
self._publish_round_outcome(exc)
|
|
except BaseException:
|
|
logger.exception("failed to publish nccl_m2n round failure")
|
|
raise
|
|
|
|
self._publish_round_outcome(None)
|
|
|
|
def _publish_round_outcome(self, failure: BaseException | None) -> None:
|
|
"""Tell non-sender trainer ranks whether worker finalization passed."""
|
|
if self._num_trainer_ranks <= 1:
|
|
return
|
|
if self._metadata_group is None:
|
|
raise RuntimeError("nccl_m2n metadata group is unavailable")
|
|
envelope = {
|
|
"ok": failure is None,
|
|
"error": (
|
|
None if failure is None else f"{type(failure).__name__}: {failure}"
|
|
),
|
|
}
|
|
for rank in range(1, self._num_trainer_ranks):
|
|
self._metadata_group.send_obj(envelope, rank)
|
|
# Keep rank zero's TCPStore alive until every trainer consumed its
|
|
# outcome. This also keeps the per-rank message counters aligned.
|
|
expected_ack = {"phase": "round_outcome_consumed"}
|
|
for rank in range(1, self._num_trainer_ranks):
|
|
ack = self._metadata_group.recv_obj(rank)
|
|
if ack != expected_ack:
|
|
raise RuntimeError(
|
|
f"trainer rank {rank} sent an invalid round-outcome acknowledgement"
|
|
)
|
|
|
|
def _receive_round_outcome(self) -> None:
|
|
"""Wait until trainer rank zero observed every worker's finish RPC."""
|
|
if self._metadata_group is None:
|
|
raise RuntimeError("nccl_m2n metadata group is unavailable")
|
|
outcome = self._metadata_group.recv_obj(0)
|
|
self._metadata_group.send_obj({"phase": "round_outcome_consumed"}, 0)
|
|
if not isinstance(outcome, Mapping) or set(outcome) != {"ok", "error"}:
|
|
raise RuntimeError("trainer rank zero sent an invalid round outcome")
|
|
ok = outcome["ok"]
|
|
error = outcome["error"]
|
|
if not isinstance(ok, bool) or ok != (error is None):
|
|
raise RuntimeError("trainer rank zero sent an invalid round outcome")
|
|
if not ok:
|
|
if not isinstance(error, str) or not error:
|
|
raise RuntimeError("trainer rank zero sent an invalid round error")
|
|
raise RuntimeError(
|
|
"nccl_m2n worker update failed on trainer rank zero: " + error
|
|
)
|
|
|
|
def _send(self) -> None:
|
|
"""Reshard each local shard into its worker-planned destination."""
|
|
assert self.group is not None
|
|
assert self._dst_mesh is not None
|
|
assert self.source is not None
|
|
m2n = self._m2n
|
|
stream = torch.cuda.current_stream()
|
|
|
|
device_index = torch.accelerator.current_device_index()
|
|
# Optimizer work may have used another CUDA stream. Make prior writes
|
|
# visible before a provider materializes current-stream local shards.
|
|
torch.accelerator.synchronize(device_index)
|
|
inflight_buffers: list[Any] = []
|
|
sent = 0
|
|
for index, (name, buffer) in enumerate(self.source):
|
|
if index >= len(self._metas):
|
|
raise RuntimeError(
|
|
"weight source yielded more parameters than metadata() "
|
|
f"declared (first extra parameter: '{name}')"
|
|
)
|
|
meta = self._metas[index]
|
|
if name != meta.name:
|
|
raise RuntimeError(
|
|
f"weight source yielded '{name}' at position {index}, but "
|
|
f"its metadata declared '{meta.name}'; the source must "
|
|
"iterate in a stable order"
|
|
)
|
|
validate_local_buffer(m2n, meta, buffer)
|
|
src_mesh, src_placements = resolve_layout(
|
|
meta.source_layout.mesh, meta.source_layout.placements
|
|
)
|
|
dst_mesh, dst_placements = resolve_layout(
|
|
self._dst_mesh, self._dst_placements[index]
|
|
)
|
|
raw_pointer_args = {}
|
|
if isinstance(buffer, int):
|
|
raw_pointer_args = {
|
|
"src_local_shape": expected_local_shape(meta),
|
|
"src_dtype": meta.dtype,
|
|
}
|
|
m2n.reshard(
|
|
buffer,
|
|
None,
|
|
self.group,
|
|
stream,
|
|
src_mesh=to_mesh(m2n, src_mesh),
|
|
src_placements=to_placements(m2n, src_placements),
|
|
dst_mesh=to_mesh(m2n, dst_mesh),
|
|
dst_placements=to_placements(m2n, dst_placements),
|
|
handle=self._handle,
|
|
**raw_pointer_args,
|
|
)
|
|
# Let allocator-aware buffers recycle storage in stream order.
|
|
# Other object owners must stay alive until the final synchronize;
|
|
# raw-pointer lifetime remains the source's responsibility.
|
|
record_stream = getattr(buffer, "record_stream", None)
|
|
if callable(record_stream):
|
|
record_stream(stream)
|
|
elif not isinstance(buffer, int):
|
|
inflight_buffers.append(buffer)
|
|
sent += 1
|
|
|
|
if sent != len(self._metas):
|
|
missing = self._metas[sent].name
|
|
raise RuntimeError(
|
|
f"weight source yielded {sent} parameters, but metadata() "
|
|
f"declared {len(self._metas)} (first missing: '{missing}')"
|
|
)
|
|
|
|
# As in the other weight-transfer backends, wait before returning
|
|
# rather than relying on same-stream ordering. The caller may then
|
|
# safely mutate parameters or continue work on another stream.
|
|
stream.synchronize()
|
|
|
|
def shutdown(self) -> None:
|
|
"""Finish pending GPU work and release M2N communication state."""
|
|
handle, self._handle = self._handle, None
|
|
group, self.group = self.group, None
|
|
self._metadata_group = None
|
|
self._dst_placements = []
|
|
self._dst_mesh = None
|
|
self._m2n = None
|
|
|
|
first_error: BaseException | None = None
|
|
|
|
def attempt(action: Any, message: str) -> None:
|
|
nonlocal first_error
|
|
try:
|
|
action()
|
|
except BaseException as exc:
|
|
if first_error is None:
|
|
first_error = exc
|
|
logger.exception(message)
|
|
|
|
if handle is not None or group is not None:
|
|
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
|
|
if handle is not None:
|
|
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
|
|
if group is not None:
|
|
attempt(group.abort, "failed to abort nccl_m2n communicator")
|
|
if first_error is not None:
|
|
raise first_error
|