[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers

- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs,
  moriio, hisparse, example and hidden-states connectors
- Reduce weight_transfer to the base API used by model loading
- Remove kv_offload tiering, simple_kv_offload, ec cpu/example,
  snapshot command, engine snapshot server
- Remove transformers model fallback, dsv4 tokenizers, dead platforms
  cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn
  ops, dead layers (short_conv, fused_embed_norm, indexer_topk,
  relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar
- Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant
- Rebuild parser hubs around Qwen3 adapters only (36 dead adapters)
- Remove dead tests with the code they covered

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 02:19:03 +07:00
co-authored by Hinata
parent 54a396586e
commit 2b544b7ba0
353 changed files with 29 additions and 121269 deletions
@@ -1,459 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""IPC-based weight transfer engine using CUDA IPC for communication."""
import pickle
from dataclasses import asdict, dataclass
from typing import TYPE_CHECKING, Any, ClassVar, Self
import pybase64 as base64
import torch
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
from vllm import envs
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
TrainerInitInfo,
TrainerWeightTransferEngine,
VLLMWeightSyncClient,
WeightSource,
WeightTransferEngine,
WeightTransferInitInfo,
WeightTransferUpdateInfo,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
PackedBufferImporter,
packed_ipc_consumer,
packed_ipc_producer,
)
@dataclass
class IPCWeightTransferInitInfo(WeightTransferInitInfo):
"""Worker-side init info for IPC weight transfer. No rendezvous needed.
`packed` is a must-agree wire param: the trainer ships it here at the init
handshake so the worker decodes with the same setting the trainer encoded
with. The consumer rebuilds from the IPC handle + `tensor_sizes`, so it does
not need the buffer size (producer-only)."""
packed: bool = False
@dataclass
class IPCTrainerInitInfo(TrainerInitInfo):
"""Trainer-side init info for IPC weight transfer. No rendezvous needed;
`rank` (from `TrainerInitInfo`) identifies this trainer process — rank 0
ships the merged IPC handles. All ranks still join the handle all-gather.
`packed` / `packed_buffer_size_bytes` are the transfer's wire params. The
trainer propagates `packed` to the worker at `trainer_init` so the two sides
cannot disagree. `backend` is the factory dispatch key."""
backend: ClassVar[str] = "ipc"
packed: bool = False
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
@dataclass
class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Per-round update info for the IPC weight transfer backend."""
names: list[str]
dtype_names: list[str]
shapes: list[list[int]]
ipc_handles: list[dict[str, tuple]] | dict[str, tuple] | None = None
"""IPC handles mapping physical GPU UUID to rebuild_cuda_tensor args.
For non-packed mode: list of per-parameter handle dicts.
For packed mode: single handle dict for the packed buffer."""
ipc_handles_pickled: str | None = None
"""Base64-encoded pickled IPC handles, used for HTTP transport."""
tensor_sizes: list[int] | None = None
"""Per-parameter sizes in bytes within the packed buffer.
Required when packed=True, unused otherwise."""
def __post_init__(self):
if self.ipc_handles_pickled is not None:
if self.ipc_handles is not None:
raise ValueError(
"Cannot specify both `ipc_handles` and `ipc_handles_pickled`"
)
if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION:
raise ValueError(
"Refusing to deserialize `ipc_handles_pickled` without "
"VLLM_ALLOW_INSECURE_SERIALIZATION=1"
)
self.ipc_handles = pickle.loads(base64.b64decode(self.ipc_handles_pickled))
self.ipc_handles_pickled = None
if self.ipc_handles is None:
raise ValueError(
"Either `ipc_handles` or `ipc_handles_pickled` must be provided"
)
num_params = len(self.names)
if len(self.dtype_names) != num_params:
raise ValueError(
f"`dtype_names` should be of the same size as `names`: "
f"got {len(self.dtype_names)} and {len(self.names)}"
)
if len(self.shapes) != num_params:
raise ValueError(
f"`shapes` should be of the same size as `names`: "
f"got {len(self.shapes)} and {len(self.names)}"
)
# Unpacked transfers carry a per-parameter handle dict (a list); packed
# transfers carry a single dict for the whole buffer, so the list check
# only applies to the list case.
if isinstance(self.ipc_handles, list) and len(self.ipc_handles) != num_params:
raise ValueError(
f"`ipc_handles` should be of the same size as `names`: "
f"got {len(self.ipc_handles)} and {len(self.names)}"
)
class IPCWeightTransferEngine(
WeightTransferEngine[IPCWeightTransferInitInfo, IPCWeightTransferUpdateInfo]
):
"""Weight transfer engine using CUDA IPC between trainer and workers.
This implementation uses CUDA IPC to transfer weights from the trainer (rank 0)
to all inference workers in a process group. IPC handles are used to share
memory between processes on the same node.
"""
# Define backend-specific dataclass types
init_info_cls = IPCWeightTransferInitInfo
update_info_cls = IPCWeightTransferUpdateInfo
def __init__(
self,
config: WeightTransferConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
super().__init__(config, vllm_config, device, model)
# Set from the trainer-supplied init info at the handshake; defaults are
# only for the (unreachable) receive-before-init case.
self.packed = False
# Shared across all chunks of every packed transfer this engine
# receives; see PackedBufferImporter for the refcount contract.
self._packed_importer = PackedBufferImporter()
def init_transfer_engine(self, init_info: IPCWeightTransferInitInfo) -> None:
"""Initialize the weight transfer mechanism. No data-plane rendezvous is
needed for IPC; this just records the trainer-supplied wire params so the
worker decodes exactly as the trainer encoded.
Args:
init_info: IPC initialization info (carries `packed`).
"""
self.packed = init_info.packed
def start_weight_update(self) -> None:
"""Initialize layerwise reloading for the incoming checkpoint weights."""
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
initialize_layerwise_reload(self.model)
def finish_weight_update(self) -> None:
"""Finalize layerwise reloading after all weights have been received."""
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
)
finalize_layerwise_reload(self.model, self.model_config)
# Every reduce_tensor call is a fresh export with its own refcount
# slot, so releasing once per update always balances this update's
# export and lets the trainer reclaim its staging buffer. Callers
# that skip finish are still covered by the replace-on-next-export
# path inside the importer.
self._packed_importer.close()
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
"""Receive weights from the trainer via CUDA IPC handles and load them.
Whether the transfer is packed is read from `self.packed`, set at the
init handshake from the trainer's init info, so it is guaranteed to
match how the trainer encoded.
Args:
update_info: IPC update info containing parameter names, dtypes, shapes,
and IPC handles. Each IPC handle is a mapping between physical
GPU UUID and the rebuild_cuda_tensor args tuple.
"""
# Use the worker's assigned device rather than the ambient current
# device: the receive path is no longer wrapped in
# `with torch.device(self.device)` by the caller, so the current device
# is not guaranteed to match self.device. The IPC tensors must be
# rebuilt on the device the model lives on.
device_index = self.device.index
if self.packed:
if update_info.tensor_sizes is None:
raise ValueError("`tensor_sizes` is required when packed=True")
assert isinstance(update_info.ipc_handles, dict)
weights = packed_ipc_consumer(
ipc_handle=update_info.ipc_handles,
names=update_info.names,
shapes=update_info.shapes,
dtype_names=update_info.dtype_names,
tensor_sizes=update_info.tensor_sizes,
device_index=device_index,
importer=self._packed_importer,
)
else:
assert isinstance(update_info.ipc_handles, list)
weights = []
for name, ipc_handle in zip(
update_info.names,
update_info.ipc_handles,
):
props = torch.cuda.get_device_properties(device_index)
physical_gpu_id = str(props.uuid)
if physical_gpu_id not in ipc_handle:
raise ValueError(
f"IPC handle not found for GPU UUID "
f"{physical_gpu_id}. "
f"Available UUIDs: {list(ipc_handle.keys())}"
)
args = ipc_handle[physical_gpu_id]
list_args = list(args)
# Index 6 of the args from reduce_tensor is the device_index.
# We need to overwrite it with the receiver's device index.
list_args[6] = device_index
weight = rebuild_cuda_tensor(*list_args)
weights.append((name, weight))
from vllm.model_executor.model_loader.mtp_validation import (
disable_mtp_completeness_check,
)
with disable_mtp_completeness_check():
self.model.load_weights(weights)
def shutdown(self) -> None:
self._packed_importer.close()
class IPCTrainerWeightTransferEngine(TrainerWeightTransferEngine[IPCTrainerInitInfo]):
"""Trainer-side CUDA IPC weight transfer engine.
Called on every trainer rank. For multi-rank (e.g. FSDP) trainers all ranks
iterate the source (materializing each tensor) and contribute to the
IPC-handle all-gather; only the sender (rank 0) ships the merged handles to
the inference side. IPC transfer is straight-line (no concurrent broadcast
like NCCL): `update_weights` *is* the transfer, and it rides the client, so
it no-ops on non-senders.
`packed` / `packed_buffer_size_bytes` come from `IPCTrainerInitInfo`; the
sender propagates `packed` to the worker at `trainer_init`.
"""
init_info_cls = IPCTrainerInitInfo
def __init__(
self,
*,
client: VLLMWeightSyncClient,
source: WeightSource,
is_sender: bool = True,
packed: bool = False,
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
) -> None:
super().__init__(client=client, source=source, is_sender=is_sender)
self.packed = packed
self.packed_buffer_size_bytes = packed_buffer_size_bytes
self.device_index = torch.accelerator.current_device_index()
self.gpu_uuid = str(torch.cuda.get_device_properties(self.device_index).uuid)
@classmethod
def trainer_init(
cls,
init_info: IPCTrainerInitInfo,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
) -> Self:
if source is None:
raise ValueError("IPC trainer weight transfer requires a WeightSource.")
engine = cls(
client=client,
source=source,
is_sender=init_info.is_sender,
packed=init_info.packed,
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
)
# IPC needs no data-plane rendezvous. The sender ships the must-agree
# `packed` flag so the worker decodes exactly as this trainer encodes.
if engine.is_sender:
engine.client.init_weight_transfer_engine({"packed": init_info.packed})
return engine
def send_weights(self) -> None:
assert self.source is not None
source = self.source
if self.is_sender:
self.client.start_weight_update()
# Unpacked returns strong refs to its IPC-shared copies; they must stay
# alive across `finish_weight_update` too.
weight_refs = self._send(source)
if self.is_sender:
self.client.finish_weight_update()
self._post_send_sync()
del weight_refs
# ---- data plane (runs on all ranks; only the sender ships) ----
def _send(self, source: WeightSource) -> list[torch.Tensor] | None:
if self.packed:
self._send_packed(source)
return None
return self._send_unpacked(source)
def _all_gather_and_merge_handles(
self,
handles: list[dict[str, tuple]],
) -> list[dict[str, tuple]]:
"""All-gather and merge IPC handle dicts across ranks in one call.
Each rank contributes a list of {gpu_uuid: ipc_args} dicts (one
per parameter or one per chunk). A single all_gather_object
collects every rank's full list, then the sender merges per-index so
each dict maps every GPU UUID to its args.
The all-gather runs over the *default* process group; this assumes the
default group is exactly the set of colocated trainer ranks and that the
sender is a member. No-op (returns handles unchanged) when no distributed
group exists.
"""
if (
not torch.distributed.is_initialized()
or torch.distributed.get_world_size() == 1
):
return handles
world_size = torch.distributed.get_world_size()
gathered: list[list[dict[str, tuple]] | None] = [None] * world_size
torch.distributed.all_gather_object(gathered, handles)
torch.distributed.barrier()
torch.cuda.synchronize()
if self.is_sender:
merged: list[dict[str, tuple]] = []
for param_idx in range(len(handles)):
m: dict[str, tuple] = {}
for rank_handles in gathered:
if rank_handles is not None:
m.update(rank_handles[param_idx])
merged.append(m)
return merged
return [{} for _ in handles]
@staticmethod
def _post_send_sync() -> None:
"""Barrier + ipc_collect after a send; no-op if single-GPU."""
if (
torch.distributed.is_initialized()
and torch.distributed.get_world_size() > 1
):
torch.distributed.barrier()
torch.cuda.ipc_collect()
def _send_unpacked(self, source: WeightSource) -> list[torch.Tensor]:
"""Iterate the source, build one IPC handle per param, all-gather the
handles across ranks, and (sender) ship them in one update call.
Returns the strong refs to every contiguous copy. reduce_tensor's args
do NOT keep storage alive, and non-contiguous inputs allocate fresh
storage in .contiguous(); the caller must keep these alive until the
post-send barrier (past `finish`) so the consumer's IPC views stay valid.
"""
names: list[str] = []
dtype_names: list[str] = []
shapes: list[list[int]] = []
ipc_handles: list[dict[str, tuple]] = []
weight_refs: list[torch.Tensor] = []
for name, tensor in source:
# FSDP shards were gathered by the source; ensure contiguity here.
names.append(name)
dtype_names.append(str(tensor.dtype).split(".")[-1])
shapes.append(list(tensor.shape))
weight = tensor.detach().contiguous()
weight_refs.append(weight)
_, ipc_args = reduce_tensor(weight)
ipc_handles.append({self.gpu_uuid: ipc_args})
ipc_handles = self._all_gather_and_merge_handles(ipc_handles)
self._do_send(
names=names,
dtype_names=dtype_names,
shapes=shapes,
ipc_handles=ipc_handles,
)
return weight_refs
def _send_packed(self, source: WeightSource) -> None:
"""Send weights in bounded-memory chunks (packed mode)."""
for chunk in packed_ipc_producer(
iterator=iter(source),
gpu_uuid=self.gpu_uuid,
post_iter_func=lambda item: item[1],
buffer_size_bytes=self.packed_buffer_size_bytes,
):
ipc_handle = self._all_gather_and_merge_handles([chunk.ipc_handle])[0]
self._do_send(
names=chunk.names,
dtype_names=chunk.dtype_names,
shapes=chunk.shapes,
ipc_handles=ipc_handle,
tensor_sizes=chunk.tensor_sizes,
)
# Per-chunk barrier: the producer reuses a single IPC buffer across
# chunks, but only the sender waits for the consumers (via _do_send).
# Without syncing every rank here, non-sender ranks race ahead and
# overwrite their buffer while their colocated worker is still
# reading the current chunk, silently corrupting the transfer.
self._post_send_sync()
def _do_send(
self,
names: list[str],
dtype_names: list[str],
shapes: list[list[int]],
ipc_handles: list[dict[str, tuple]] | dict[str, tuple],
tensor_sizes: list[int] | None = None,
) -> None:
"""Build one update payload and ship it via the client. Only the sender
ships (non-sender ranks already contributed to the handle all-gather).
Emits raw `ipc_handles`; transports that cannot carry them natively
(HTTP/JSON) pickle them in their client (see `HTTPVLLMWeightSyncClient`).
"""
if not self.is_sender:
return
update_fields: dict[str, Any] = {
"names": names,
"dtype_names": dtype_names,
"shapes": shapes,
"ipc_handles": ipc_handles,
}
if tensor_sizes is not None:
update_fields["tensor_sizes"] = tensor_sizes
update_info = IPCWeightTransferUpdateInfo(**update_fields)
self.client.update_weights(asdict(update_info))
@@ -1,864 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared helpers for the NCCL M2N (`nccl_m2n`) weight transfer backend.
M2N reshards a tensor between two disjoint meshes of ranks that live in one
communicator: the trainer occupies ranks `[0, T)` and the inference workers
`[T, T + N)`. This module holds everything both sides need — the optional
runtime import, layout descriptors, and the conversion to `nccl.m2n` types —
so the engine module stays about the transfer itself.
"""
import hashlib
import json
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
import torch
from vllm.distributed.weight_transfer.base import ParamMeta
from vllm.utils.nccl import unpinned_nccl_env
if TYPE_CHECKING:
from vllm.distributed.utils import StatelessProcessGroup
# Placement code for a mesh axis that replicates. Any other (non-negative) code
# is the tensor dim that axis shards. Ints rather than `nccl.m2n` objects so
# layouts survive the JSON init handshake without a custom encoder.
REPLICATE = -1
# `NCCL_RESHARD_MAX_TENSOR_DIMS`; 4-D and higher are rejected by the library.
MAX_TENSOR_DIMS = 3
MESH_NDIMS = 2
# Destination mesh convention. Keeping these roles named avoids scattering
# positional axis assumptions through the worker-side planner.
DESTINATION_REPLICA_AXIS = 0
DESTINATION_SHARD_AXIS = 1
# Conservative capacities accepted by the supported nccl_m2n algorithms. They
# are not exposed by its Python API yet; remove these mirrors once nccl_m2n
# provides communication-free plan validation.
MAX_SOURCE_SHARDS = 16
MAX_DEST_SHARDS = 64
# Increment when the JSON init payload changes incompatibly.
M2N_WIRE_SCHEMA_VERSION = 2
SUPPORTED_DTYPES: frozenset[torch.dtype] = frozenset(
dtype
for dtype in (
getattr(torch, name, None)
for name in (
"int8",
"uint8",
"float8_e4m3fn",
"float8_e5m2",
"float16",
"bfloat16",
"int32",
"uint32",
"float32",
"int64",
"uint64",
"float64",
)
)
if dtype is not None
)
_IMPORT_HINT = (
"The nccl_m2n weight transfer backend requires the `nccl-extensions` "
"package (and its `nccl4py` dependency), which is not installed by vLLM. "
"See https://github.com/NVIDIA/nccl-extensions for build "
"and install instructions. It needs NCCL 2.30.5 or newer."
)
def prepare_m2n_local_runtime(m2n: Any, max_cta: int | None) -> tuple[Any, int, Any]:
"""Prepare rank-local resources before any rank enters NCCL init."""
import nccl.core as nccl
device = torch.accelerator.current_device_index()
if not isinstance(device, int) or isinstance(device, bool) or device < 0:
raise RuntimeError(f"invalid current accelerator device index: {device!r}")
handle = m2n.Handle.create(m2n.Config(max_cta=max_cta))
return nccl, device, handle
def create_m2n_communicator(
nccl: Any,
group: "StatelessProcessGroup",
unique_id_bytes: bytes | None,
) -> Any:
"""Create the M2N data-plane communicator through NCCL4Py."""
if unique_id_bytes is None:
envelope: dict[str, bytes | str | None] | None = None
if group.rank == 0:
try:
envelope = {
"unique_id": bytes(nccl.get_unique_id().as_bytes),
"error": None,
}
except Exception as exc:
envelope = {
"unique_id": None,
"error": f"{type(exc).__name__}: {exc}",
}
envelope = group.broadcast_obj(envelope, src=0)
if not isinstance(envelope, Mapping) or set(envelope) != {"unique_id", "error"}:
raise RuntimeError("invalid NCCL4Py unique-id envelope")
error = envelope["error"]
received_unique_id = envelope["unique_id"]
if error is not None:
if not isinstance(error, str) or not error:
raise RuntimeError("invalid NCCL4Py unique-id error")
raise RuntimeError(f"NCCL4Py unique-id generation failed: {error}")
if not isinstance(received_unique_id, bytes):
raise RuntimeError("NCCL4Py unique-id broadcast did not return bytes")
unique_id_bytes = received_unique_id
unique_id = nccl.UniqueId.from_bytes(unique_id_bytes)
with unpinned_nccl_env():
return nccl.Communicator.init(
nranks=group.world_size,
rank=group.rank,
unique_id=unique_id,
)
def import_m2n() -> Any:
"""Import `nccl.m2n` lazily, with an actionable error when it is missing.
Deferred so that importing vLLM — or any other weight transfer backend —
never requires the m2n runtime to be present.
"""
try:
import nccl.m2n as m2n
except ImportError as e:
raise ImportError(f"{_IMPORT_HINT} (import failed: {e})") from e
return m2n
@dataclass(frozen=True)
class M2NMesh:
"""One side's rank topology — pure topology, no tensor placement.
Mirrors `ncclMesh_t`: a 2-axis mesh owning the contiguous rank interval
`[start_rank, start_rank + dims[0] * dims[1])`. There is no 1-D mesh; a
single-axis topology is spelled with a second axis of size 1.
A model may use a different factorization for each tensor. For example,
dense weights can use a DP x TP mesh while expert weights use EDP x EP.
"""
dims: tuple[int, int]
start_rank: int
def __post_init__(self) -> None:
object.__setattr__(self, "dims", tuple(self.dims))
if len(self.dims) != MESH_NDIMS or any(
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
for dim in self.dims
):
raise ValueError(f"mesh dims must be {MESH_NDIMS} positive ints")
if (
not isinstance(self.start_rank, int)
or isinstance(self.start_rank, bool)
or self.start_rank < 0
):
raise ValueError(
f"mesh start_rank must be non-negative, got {self.start_rank}"
)
@property
def size(self) -> int:
return self.dims[0] * self.dims[1]
# A tensor's placement over its side's mesh: one code per mesh axis
# (`REPLICATE`, or the tensor dim that axis shards), or `REPLICATED` for a
# tensor every rank holds in full.
Placements = tuple[int, int]
REPLICATED: Placements | None = None
@dataclass(frozen=True)
class M2NLayout:
"""A tensor's mesh and placement on one side of a transfer."""
mesh: M2NMesh
placements: Placements | None
def __post_init__(self) -> None:
if not isinstance(self.mesh, M2NMesh):
raise TypeError("M2N layout mesh must be an M2NMesh")
if self.placements is not None:
object.__setattr__(self, "placements", tuple(self.placements))
def check_placements(placements: Placements, context: str = "placements") -> None:
"""Reject placement pairs nccl_m2n cannot express."""
if len(placements) != MESH_NDIMS:
raise ValueError(f"{context} must have {MESH_NDIMS} entries")
if any(not isinstance(code, int) or isinstance(code, bool) for code in placements):
raise TypeError(f"{context} must contain integer placement codes")
invalid = [code for code in placements if code < REPLICATE]
if invalid:
raise ValueError(
f"{context} contains invalid placement code {invalid[0]}; valid "
f"codes are {REPLICATE} (Replicate) and non-negative tensor dimensions"
)
num_sharded = sum(code != REPLICATE for code in placements)
if num_sharded == 0:
raise ValueError(
f"{context}: a fully replicated tensor is carried as REPLICATED, "
"not as two REPLICATE axes"
)
if num_sharded == MESH_NDIMS:
raise ValueError(
f"{context}: nccl_m2n needs one REPLICATE mesh axis, but "
f"{placements} shards both"
)
def resolve_layout(
mesh: M2NMesh,
placements: Placements | None,
context: str = "placements",
) -> tuple[M2NMesh, Placements]:
"""Pair one tensor's placement with the mesh m2n should see for it.
A replicated tensor needs a size-1 mesh axis to carry a no-op shard, since
m2n has no `{REPLICATE, REPLICATE}`. It is therefore described over the
*same rank interval* re-factored as `(size, 1)`. That re-factoring is sound
precisely because replication is order-independent — every rank holds the
whole tensor, so it does not matter that `(a, b)` and `(size, 1)` walk the
interval in a different order. A sharded tensor keeps its side's own
factorization, where rank order decides who owns which shard.
"""
if placements is None:
return M2NMesh((mesh.size, 1), mesh.start_rank), (REPLICATE, 0)
check_placements(placements, context)
return mesh, placements
def validate_layout(
mesh: M2NMesh,
placements: Placements,
shape: Sequence[int],
context: str,
) -> tuple[int, ...]:
"""Validate one resolved layout and return its required local shape."""
local_shape = list(shape)
for axis, tensor_dim in enumerate(placements):
if tensor_dim == REPLICATE:
continue
if tensor_dim >= len(shape):
raise ValueError(
f"{context} shards tensor dim {tensor_dim}, but parameter has "
f"rank {len(shape)}"
)
factor = mesh.dims[axis]
if shape[tensor_dim] % factor:
raise ValueError(
f"{context} shards dim {tensor_dim} (size {shape[tensor_dim]}) "
f"over {factor} ranks, which does not divide evenly"
)
local_shape[tensor_dim] //= factor
return tuple(local_shape)
def expected_local_shape(meta: "M2NParamMeta") -> tuple[int, ...]:
"""Validate a parameter's source layout and return its local shape."""
mesh, placements = resolve_layout(
meta.source_layout.mesh,
meta.source_layout.placements,
f"parameter '{meta.name}' source placements",
)
return validate_layout(
mesh, placements, meta.shape, f"parameter '{meta.name}' source layout"
)
def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> None:
"""Reject parameters nccl_m2n cannot transfer."""
if dtype not in SUPPORTED_DTYPES:
raise ValueError(
f"parameter '{name}' has dtype {dtype}, which nccl_m2n does not "
f"support. Supported: {sorted(str(d) for d in SUPPORTED_DTYPES)}"
)
if not 1 <= len(shape) <= MAX_TENSOR_DIMS:
raise ValueError(
f"parameter '{name}' has rank {len(shape)}; nccl_m2n supports "
f"rank 1..{MAX_TENSOR_DIMS}"
)
def _shard_spec(mesh: M2NMesh, placements: Placements) -> tuple[int, int]:
return next(
(code, mesh.dims[axis])
for axis, code in enumerate(placements)
if code != REPLICATE
)
def _max_partition_overlaps(
extent: int,
partitions: int,
other_partitions: int,
) -> int:
"""Return the maximum other partitions intersected by any one partition."""
if extent == 0:
return 0
width = extent // partitions
other_width = extent // other_partitions
max_overlaps = 0
for index in range(partitions):
start = index * width
end = start + width
first_other = start // other_width
last_other = (end - 1) // other_width
max_overlaps = max(max_overlaps, last_other - first_other + 1)
return max_overlaps
def check_plan_limits(
src: tuple[M2NMesh, Placements],
dst: tuple[M2NMesh, Placements],
name: str,
shape: Sequence[int],
) -> None:
"""Reject plans whose per-shard fan-in or fan-out exceeds M2N capacity."""
src_dim, src_shards = _shard_spec(*src)
dst_dim, dst_shards = _shard_spec(*dst)
if any(extent == 0 for extent in shape):
max_sources = max_targets = 0
elif src_dim != dst_dim:
max_sources = src_shards
max_targets = dst_shards
else:
extent = shape[src_dim]
max_sources = _max_partition_overlaps(extent, dst_shards, src_shards)
max_targets = _max_partition_overlaps(extent, src_shards, dst_shards)
if max_sources > MAX_SOURCE_SHARDS:
raise ValueError(
f"parameter '{name}' would feed up to {max_sources} source shards "
f"into one destination shard, over nccl_m2n MAX_SOURCES="
f"{MAX_SOURCE_SHARDS}"
)
if max_targets > MAX_DEST_SHARDS:
raise ValueError(
f"parameter '{name}' would fan one source shard out to up to "
f"{max_targets} destination shards, over nccl_m2n "
f"MAX_TARGETS={MAX_DEST_SHARDS}"
)
def validate_transfer_plan(
metas: Sequence["M2NParamMeta"],
dst_mesh: M2NMesh,
dst_placements: Sequence[Placements | None],
) -> None:
"""Validate the complete ordered source-to-destination transfer plan."""
if len(dst_placements) != len(metas):
raise ValueError(
f"destination plan has {len(dst_placements)} parameters, expected "
f"{len(metas)}"
)
for meta, placement in zip(metas, dst_placements):
check_transferable(meta.name, meta.dtype, meta.shape)
src = resolve_layout(
meta.source_layout.mesh,
meta.source_layout.placements,
f"parameter '{meta.name}' source placements",
)
validate_layout(*src, meta.shape, f"parameter '{meta.name}' source layout")
dst = resolve_layout(
dst_mesh,
placement,
f"parameter '{meta.name}' destination placements",
)
validate_layout(*dst, meta.shape, f"parameter '{meta.name}' destination layout")
check_plan_limits(src, dst, meta.name, meta.shape)
@dataclass(frozen=True)
class M2NParamMeta(ParamMeta):
"""`ParamMeta` extended with how the trainer places this tensor.
The base class carries only name / dtype / full shape, which is not enough
to plan a reshard. `source_layout` carries both the per-parameter mesh and
its placement on that mesh.
"""
source_layout: M2NLayout
def __post_init__(self) -> None:
if not isinstance(self.source_layout, M2NLayout):
raise TypeError("source_layout must be an M2NLayout")
@dataclass(frozen=True)
class M2NWireParam:
"""JSON-safe source metadata for one parameter in the init handshake."""
name: str
dtype_name: str
shape: tuple[int, ...]
src_mesh_dims: tuple[int, int]
src_placements: Placements | None
def __post_init__(self) -> None:
shape = tuple(self.shape)
dims = tuple(self.src_mesh_dims)
placements = None if self.src_placements is None else tuple(self.src_placements)
object.__setattr__(self, "shape", shape)
object.__setattr__(self, "src_mesh_dims", dims)
object.__setattr__(self, "src_placements", placements)
if not isinstance(self.name, str) or not self.name:
raise ValueError("M2N wire parameter name must not be empty")
if not isinstance(self.dtype_name, str) or not self.dtype_name:
raise ValueError(f"parameter '{self.name}' has an empty wire dtype name")
if not shape or any(
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
for dim in shape
):
raise ValueError(
f"parameter '{self.name}' shape must contain positive integers"
)
M2NMesh(cast(tuple[int, int], dims), 0)
if placements is not None:
check_placements(
cast(Placements, placements),
f"parameter '{self.name}' source placements",
)
@classmethod
def from_dict(cls, value: Mapping[str, Any]) -> "M2NWireParam":
"""Parse one strict wire record from its JSON-decoded mapping."""
fields = {
"name",
"dtype_name",
"shape",
"src_mesh_dims",
"src_placements",
}
missing = fields - value.keys()
extra = value.keys() - fields
if missing or extra:
details = []
if missing:
details.append(f"missing {sorted(missing)}")
if extra:
details.append(f"unexpected {sorted(extra)}")
raise ValueError(f"invalid M2N wire parameter: {', '.join(details)}")
return cls(**value)
def to_dict(self) -> dict[str, Any]:
"""Serialize this record using only JSON-compatible values."""
return {
"name": self.name,
"dtype_name": self.dtype_name,
"shape": list(self.shape),
"src_mesh_dims": list(self.src_mesh_dims),
"src_placements": (
None if self.src_placements is None else list(self.src_placements)
),
}
def source_plan_digest(params: Sequence[M2NWireParam]) -> str:
"""Return a rank-independent digest of the ordered source wire plan."""
payload = json.dumps(
{
"schema_version": M2N_WIRE_SCHEMA_VERSION,
"params": [param.to_dict() for param in params],
},
sort_keys=True,
separators=(",", ":"),
).encode()
return hashlib.sha256(payload).hexdigest()
def complete_plan_digest(
source_digest: str,
dst_mesh: M2NMesh,
dst_placements: Sequence[Placements | None],
) -> str:
"""Return a digest of the agreed source and destination plan."""
if not isinstance(source_digest, str) or not source_digest:
raise ValueError("source digest must be a non-empty string")
payload = json.dumps(
{
"schema_version": M2N_WIRE_SCHEMA_VERSION,
"source_digest": source_digest,
"destination_mesh": {
"dims": list(dst_mesh.dims),
"start_rank": dst_mesh.start_rank,
},
"destination_placements": [
None if placement is None else list(placement)
for placement in dst_placements
],
},
sort_keys=True,
separators=(",", ":"),
).encode()
return hashlib.sha256(payload).hexdigest()
def check_source_plan_agreement(
group: "StatelessProcessGroup",
params: Sequence[M2NWireParam],
expected_digest: str | None = None,
local_error: str | None = None,
) -> str:
"""Verify every participant entered with the same source plan."""
prepared_error = local_error
digest: str | None = None
declared: str | None = expected_digest
if prepared_error is None:
try:
digest = source_plan_digest(params)
declared = digest if expected_digest is None else expected_digest
if not isinstance(declared, str) or not declared:
raise ValueError("declared source digest must be a non-empty string")
except Exception as exc:
prepared_error = f"{type(exc).__name__}: {exc}"
elif not isinstance(prepared_error, str) or not prepared_error:
prepared_error = "TypeError: local source error must be a non-empty string"
gathered = group.all_gather_obj(
{
"phase": "source",
"digest": digest,
"declared_digest": declared,
"error": prepared_error,
}
)
errors: list[str] = []
expected_fields = {"phase", "digest", "declared_digest", "error"}
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
agreed: list[tuple[int, str, str]] = []
for rank, item in enumerate(gathered):
if not isinstance(item, Mapping) or set(item) != expected_fields:
errors.append(f"rank {rank}: invalid source preflight envelope")
continue
if item["phase"] != "source":
errors.append(f"rank {rank}: invalid source preflight phase")
continue
error = item["error"]
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid source preflight error")
else:
errors.append(f"rank {rank}: {error}")
continue
computed = item["digest"]
claimed = item["declared_digest"]
if (
not isinstance(computed, str)
or not computed
or not isinstance(claimed, str)
or not claimed
):
errors.append(f"rank {rank}: invalid source digest")
continue
agreed.append((rank, computed, claimed))
if errors:
raise RuntimeError(
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
)
canonical = agreed[0][1]
for rank, computed, claimed in agreed:
if computed != claimed:
errors.append(
f"rank {rank}: computed digest {computed} does not match "
f"declared digest {claimed}"
)
if computed != canonical:
errors.append(f"rank {rank}: source digest disagrees with rank 0")
if errors:
raise RuntimeError(
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
)
return canonical
def check_complete_plan_agreement(
group: "StatelessProcessGroup",
source_digest: str,
dst_mesh: M2NMesh,
dst_placements: Sequence[Placements | None],
local_error: str | None = None,
) -> str:
"""Require every rank to accept the same complete transfer plan."""
prepared_error = local_error
digest: str | None = None
if prepared_error is None:
try:
digest = complete_plan_digest(source_digest, dst_mesh, dst_placements)
except Exception as exc:
prepared_error = f"{type(exc).__name__}: {exc}"
elif not isinstance(prepared_error, str) or not prepared_error:
prepared_error = "TypeError: local plan error must be a non-empty string"
gathered = group.all_gather_obj(
{"phase": "complete_plan", "digest": digest, "error": prepared_error}
)
errors: list[str] = []
valid: list[tuple[int, str]] = []
expected_fields = {"phase", "digest", "error"}
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
for rank, item in enumerate(gathered):
if (
not isinstance(item, Mapping)
or set(item) != expected_fields
or item.get("phase") != "complete_plan"
):
errors.append(f"rank {rank}: invalid complete-plan envelope")
continue
error = item["error"]
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid complete-plan error")
else:
errors.append(f"rank {rank}: {error}")
continue
rank_digest = item["digest"]
if not isinstance(rank_digest, str) or not rank_digest:
errors.append(f"rank {rank}: invalid complete-plan digest")
continue
valid.append((rank, rank_digest))
if not errors:
canonical = valid[0][1]
for rank, rank_digest in valid[1:]:
if rank_digest != canonical:
errors.append(
f"rank {rank}: complete-plan digest disagrees with rank 0"
)
if errors:
raise RuntimeError(
"nccl_m2n complete-plan preflight rejected before first reshard: "
+ "; ".join(errors)
)
return valid[0][1]
def check_runtime_ready_agreement(
group: "StatelessProcessGroup", local_error: str | None
) -> None:
"""Require every rank to prepare local resources before NCCL init."""
prepared_error = local_error
if prepared_error is not None and (
not isinstance(prepared_error, str) or not prepared_error
):
prepared_error = "TypeError: local runtime error must be a non-empty string"
gathered = group.all_gather_obj({"phase": "runtime_ready", "error": prepared_error})
errors: list[str] = []
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
for rank, item in enumerate(gathered):
if (
not isinstance(item, Mapping)
or set(item) != {"phase", "error"}
or item.get("phase") != "runtime_ready"
):
errors.append(f"rank {rank}: invalid runtime readiness envelope")
continue
error = item.get("error")
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid runtime readiness error")
else:
errors.append(f"rank {rank}: {error}")
if errors:
raise RuntimeError(
"nccl_m2n local runtime preflight failed before NCCL init: "
+ "; ".join(errors)
)
def check_data_plane_agreement(
group: "StatelessProcessGroup",
unique_id_bytes: bytes | None,
local_error: str | None = None,
) -> None:
"""Agree on the NCCL rendezvous identity before NCCL init."""
prepared_error = local_error
uid_digest: str | None = None
if prepared_error is None:
try:
if unique_id_bytes is not None:
if not isinstance(unique_id_bytes, bytes):
raise TypeError("NCCL unique id must decode to bytes")
uid_digest = hashlib.sha256(unique_id_bytes).hexdigest()
except Exception as exc:
prepared_error = f"{type(exc).__name__}: {exc}"
elif not isinstance(prepared_error, str) or not prepared_error:
prepared_error = "TypeError: local data-plane error must be non-empty"
gathered = group.all_gather_obj(
{
"phase": "data_plane",
"mode": "uid" if unique_id_bytes is not None else "tcp",
"uid_digest": uid_digest,
"error": prepared_error,
}
)
expected_fields = {
"phase",
"mode",
"uid_digest",
"error",
}
errors: list[str] = []
valid: list[tuple[int, str, str | None]] = []
if len(gathered) != group.world_size:
errors.append(
"control-plane all-gather returned "
f"{len(gathered)} records for world size {group.world_size}"
)
for rank, item in enumerate(gathered):
if (
not isinstance(item, Mapping)
or set(item) != expected_fields
or item.get("phase") != "data_plane"
):
errors.append(f"rank {rank}: invalid data-plane preflight envelope")
continue
error = item["error"]
if error is not None:
if not isinstance(error, str) or not error:
errors.append(f"rank {rank}: invalid data-plane preflight error")
else:
errors.append(f"rank {rank}: {error}")
continue
mode = item["mode"]
uid = item["uid_digest"]
if mode not in {"tcp", "uid"} or (mode == "uid") != isinstance(uid, str):
errors.append(f"rank {rank}: invalid data-plane identity")
continue
valid.append((rank, mode, uid))
if not errors:
_, mode, uid = valid[0]
for rank, other_mode, other_uid in valid[1:]:
if (other_mode, other_uid) != (mode, uid):
errors.append(f"rank {rank}: NCCL data-plane identity disagrees")
if errors:
raise RuntimeError(
"nccl_m2n data-plane preflight rejected before NCCL init: "
+ "; ".join(errors)
)
def validate_local_buffer(m2n: Any, meta: M2NParamMeta, buffer: Any) -> None:
"""Check source dtype/shape while delegating buffer validity to M2N."""
# A raw pointer carries no inspectable metadata. The source owns its
# correctness; _send supplies the declared dtype and local shape to M2N.
if isinstance(buffer, int):
return
mesh, placements = resolve_layout(
meta.source_layout.mesh, meta.source_layout.placements
)
actual = m2n.DistTensor(
buffer,
mesh=to_mesh(m2n, mesh),
placements=to_placements(m2n, placements),
)
declared = m2n.DistTensor(
None,
local_shape=actual.local_shape,
dtype=meta.dtype,
mesh=to_mesh(m2n, mesh),
placements=to_placements(m2n, placements),
)
if actual.dtype != declared.dtype:
raise ValueError(
f"parameter '{meta.name}' source returned dtype {actual.dtype}, "
f"but its metadata declared {declared.dtype}"
)
expected = expected_local_shape(meta)
if actual.local_shape != expected:
raise ValueError(
f"parameter '{meta.name}' source returned local shape "
f"{actual.local_shape}, but layout {meta.source_layout} implies "
f"{expected} from global shape {meta.shape}"
)
def to_mesh(m2n: Any, mesh: M2NMesh) -> Any:
return m2n.Mesh(mesh.dims, start_rank=mesh.start_rank)
def to_placements(m2n: Any, placements: Placements) -> list[Any]:
return [
m2n.Replicate() if code == REPLICATE else m2n.Shard(code) for code in placements
]
def publish_destination_placements(
comm: Any,
device: torch.device,
stream: torch.cuda.Stream,
first_worker_rank: int,
placements: "Sequence[Placements | None] | None",
num_parameters: int,
) -> list[Placements | None]:
"""Share the worker-side destination plan with every rank in the group.
The trainer must issue each reshard with the same destination the workers
use, but once destinations are per-parameter they depend on the inference
model, which only the workers can see. The first worker publishes them here,
over the shared NCCL communicator; trainer ranks pass `None` and receive
them, and the other workers pass their own so a disagreement is caught
rather than deadlocking later.
Only placements travel: the destination *mesh* is still derived from the
rank split, identically on both sides. The plan is published on the data
plane before any M2N reshard collective.
"""
replicated_sentinel = REPLICATE - 1
encoded = torch.full(
(num_parameters, MESH_NDIMS),
replicated_sentinel,
dtype=torch.int8,
device=device,
)
if comm.rank == first_worker_rank:
if placements is None or len(placements) != num_parameters:
raise ValueError(
f"publishing rank needs {num_parameters} destination placements"
)
rows = [
[replicated_sentinel] * MESH_NDIMS if placement is REPLICATED else placement
for placement in placements
]
if rows:
encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=device))
comm.broadcast(encoded, encoded, root=first_worker_rank, stream=stream)
return [
REPLICATED
if all(code == replicated_sentinel for code in row)
else cast(Placements, tuple(row))
for row in encoded.tolist()
]
@@ -1,599 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-side weight transfer engine built on NCCL M2N (`nccl_m2n`).
The trainer and the inference workers share one NCCL communicator: trainer ranks
occupy `[0, T)`, workers `[T, T + N)`. Each parameter is moved with a single
`nccl.m2n.reshard`, which redistributes it from the trainer's layout (FSDP / EP
/ arbitrary DTensor sharding) to the inference layout — so the trainer sends its
local shards and never all-gathers a full tensor, which is what the broadcast
NCCL backend forces it to do.
When an incoming checkpoint parameter maps directly to a live vLLM parameter,
M2N reshards into that worker's local model storage. Parameters whose names or
layouts cannot be resolved receive a full tensor and fall back to
`load_weights`, preserving the existing backend's loading behavior.
Both meshes participate in every reshard, in the same order, so this backend has
the same concurrency shape as the broadcast NCCL backend: the worker must be
inside `receive_weights` while the trainer is sending. Driving that from the
trainer is the trainer engine's job.
"""
from collections.abc import Iterator, Mapping
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, cast
import torch
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferEngine,
WeightTransferInitInfo,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.m2n_common import (
DESTINATION_SHARD_AXIS,
M2N_WIRE_SCHEMA_VERSION,
MESH_NDIMS,
M2NLayout,
M2NMesh,
M2NParamMeta,
M2NWireParam,
Placements,
check_complete_plan_agreement,
check_data_plane_agreement,
check_runtime_ready_agreement,
check_source_plan_agreement,
check_transferable,
create_m2n_communicator,
expected_local_shape,
import_m2n,
prepare_m2n_local_runtime,
publish_destination_placements,
resolve_layout,
source_plan_digest,
to_mesh,
to_placements,
validate_transfer_plan,
)
from vllm.distributed.weight_transfer.m2n_layout import (
M2NDestination,
resolve_parameter_destinations,
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
decode_nccl_unique_id,
worker_init_metadata_group,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
__all__ = [
"M2NWeightTransferInitInfo",
"M2NWeightTransferUpdateInfo",
"M2NWeightTransferEngine",
]
# ---------------------------------------------------------------------------
# Wire types
# ---------------------------------------------------------------------------
@dataclass(kw_only=True)
class M2NWeightTransferInitInfo(WeightTransferInitInfo):
"""Worker-side init info: the rendezvous plus the full transfer plan.
Layouts are static for the whole run, so they ride the one-time init
handshake and the per-round update info stays a list of names. Everything is
plain JSON so the HTTP control plane carries it unchanged.
"""
schema_version: int
master_address: str
master_port: int
rank_offset: int
"""First worker rank, i.e. the number of trainer ranks."""
world_size: int
"""Trainer ranks + all inference workers."""
dst_mesh_dims: list[int]
"""The inference mesh, starting at `rank_offset`. Declared rather than
derived so both sides describe the destination identically."""
source_digest: str
"""SHA-256 of the ordered, versioned source parameter plan."""
params: list[dict[str, Any]]
"""Ordered per-parameter source metadata and layouts."""
nccl_unique_id_b64: str | None = field(default=None, repr=False)
max_cta: int | None = None
def __post_init__(self) -> None:
if not isinstance(self.master_address, str) or not self.master_address:
raise ValueError("`master_address` must name the metadata rendezvous")
if (
not isinstance(self.master_port, int)
or isinstance(self.master_port, bool)
or not 0 < self.master_port < 65536
):
raise ValueError("`master_port` must be a valid metadata port")
if (
not isinstance(self.rank_offset, int)
or isinstance(self.rank_offset, bool)
or not isinstance(self.world_size, int)
or isinstance(self.world_size, bool)
or self.rank_offset < 1
or self.rank_offset >= self.world_size
):
raise ValueError(
f"`rank_offset` ({self.rank_offset}) must leave at least one "
f"trainer rank and one worker in world_size {self.world_size}"
)
num_workers = self.world_size - self.rank_offset
dst = tuple(self.dst_mesh_dims)
try:
dst_mesh = M2NMesh(cast(tuple[int, int], dst), self.rank_offset)
except (TypeError, ValueError) as exc:
raise ValueError(f"invalid `dst_mesh_dims` {dst}: {exc}") from exc
if dst_mesh.size != num_workers:
raise ValueError(
f"`dst_mesh_dims` {dst} must be {MESH_NDIMS} dims covering the "
f"{num_workers} inference workers"
)
def parse_wire_params(self) -> tuple[M2NWireParam, ...]:
"""Parse and authenticate the ordered source plan."""
if self.schema_version != M2N_WIRE_SCHEMA_VERSION:
raise ValueError(
"unsupported nccl_m2n init schema version "
f"{self.schema_version}; expected {M2N_WIRE_SCHEMA_VERSION}"
)
if not isinstance(self.params, list):
raise TypeError("`params` must be a list of wire parameter mappings")
if not isinstance(self.source_digest, str) or not self.source_digest:
raise ValueError("`source_digest` must be a non-empty string")
wire_params: list[M2NWireParam] = []
names: set[str] = set()
for index, value in enumerate(self.params):
if not isinstance(value, Mapping):
raise TypeError(
f"`params[{index}]` must be a mapping, got {type(value).__name__}"
)
param = M2NWireParam.from_dict(value)
if param.name in names:
raise ValueError(
f"nccl_m2n init params contain duplicate parameter '{param.name}'"
)
names.add(param.name)
wire_params.append(param)
parsed = tuple(wire_params)
computed_digest = source_plan_digest(parsed)
if self.source_digest != computed_digest:
raise ValueError(
"nccl_m2n init source digest does not match params: "
f"computed {computed_digest}, declared {self.source_digest}"
)
return parsed
@property
def nccl_unique_id_bytes(self) -> bytes | None:
if self.nccl_unique_id_b64 is None:
return None
return decode_nccl_unique_id(
master_address=None,
master_port=None,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="M2NWeightTransferInitInfo data plane",
)
@dataclass
class M2NWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Per-round update info: which parameters this chunk carries, in order.
Shapes, dtypes and layouts were fixed at init, so a round only needs to say
what is coming and in what order — both sides must issue their reshards in
exactly that order.
"""
names: list[str]
# ---------------------------------------------------------------------------
# Worker engine
# ---------------------------------------------------------------------------
class M2NWeightTransferEngine(
WeightTransferEngine[M2NWeightTransferInitInfo, M2NWeightTransferUpdateInfo]
):
"""Inference-side engine: receives each parameter with one reshard.
Resolvable parameters are resharded from the trainer layout directly into
each worker's live local model storage. Unresolvable parameters receive a
full tensor and use `load_weights`. In both cases, the trainer sends its
local shards without materializing a full tensor.
"""
init_info_cls = M2NWeightTransferInitInfo
update_info_cls = M2NWeightTransferUpdateInfo
# The destination plan is resolved against the model this engine was built
# for, so it cannot be retargeted at a draft model mid-flight.
supports_draft_weight_update = False
def __init__(
self,
config: WeightTransferConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
super().__init__(config, vllm_config, device, model)
self.model_update_group: Any = None
self._m2n: Any = None
self._handle: Any = None
self._metas: list[M2NParamMeta] = []
self._dst_mesh: M2NMesh | None = None
self._parameter_destinations: list[M2NDestination] = []
self._index: dict[str, int] = {}
self._uses_load_weights = False
def init_transfer_engine(self, init_info: M2NWeightTransferInitInfo) -> None:
"""Agree on source and local readiness before constructing NCCL."""
if self._handle is not None or self.model_update_group is not None:
raise RuntimeError(
"nccl_m2n weight transfer is already initialized; call "
"shutdown_weight_transfer_engine() before initializing again"
)
rendezvous_info = NCCLWeightTransferInitInfo(
master_address=init_info.master_address,
master_port=init_info.master_port,
rank_offset=init_info.rank_offset,
world_size=init_info.world_size,
)
metadata_group = worker_init_metadata_group(
rendezvous_info, self.parallel_config
)
wire_params: tuple[M2NWireParam, ...] = ()
source_error: str | None = None
try:
self._m2n = import_m2n()
wire_params = init_info.parse_wire_params()
self._prepare_source_plan(wire_params, init_info.rank_offset)
except Exception as exc:
source_error = f"{type(exc).__name__}: {exc}"
self._metas = []
check_source_plan_agreement(
metadata_group,
wire_params,
expected_digest=init_info.source_digest,
local_error=source_error,
)
destination_error: str | None = None
try:
self._prepare_destination_plan(init_info)
except Exception as exc:
destination_error = f"{type(exc).__name__}: {exc}"
self._parameter_destinations = []
unique_id_bytes: bytes | None = None
data_plane_error: str | None = None
try:
unique_id_bytes = init_info.nccl_unique_id_bytes
except Exception as exc:
data_plane_error = f"{type(exc).__name__}: {exc}"
check_data_plane_agreement(
metadata_group,
unique_id_bytes,
data_plane_error,
)
handle = None
group = None
try:
runtime_error = destination_error
nccl = None
device = None
if runtime_error is None:
try:
nccl, device, handle = prepare_m2n_local_runtime(
self._m2n, init_info.max_cta
)
except Exception as exc:
runtime_error = f"{type(exc).__name__}: {exc}"
check_runtime_ready_agreement(metadata_group, runtime_error)
assert nccl is not None and device is not None and handle is not None
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
# Keep the merged destination-placement publication unchanged: source
# agreement is independent of worker-side destination selection.
mine = [
destination.placements for destination in self._parameter_destinations
]
agreed = publish_destination_placements(
group,
self.device,
torch.cuda.current_stream(),
init_info.rank_offset,
mine,
len(self._metas),
)
complete_plan_error: str | None = None
try:
if agreed != mine:
mismatched = next(
meta.name
for meta, ours, theirs in zip(self._metas, mine, agreed)
if ours != theirs
)
raise RuntimeError(
"inference workers resolved different nccl_m2n "
f"destinations (first mismatch: '{mismatched}'); all "
"workers must run the same model and parallel config"
)
assert self._dst_mesh is not None
validate_transfer_plan(self._metas, self._dst_mesh, agreed)
except Exception as exc:
complete_plan_error = f"{type(exc).__name__}: {exc}"
assert self._dst_mesh is not None
check_complete_plan_agreement(
metadata_group,
init_info.source_digest,
self._dst_mesh,
agreed,
complete_plan_error,
)
except BaseException:
if group is not None:
try:
# Transport cleanup only; reload validity belongs to the
# model-update lifecycle, not this backend.
group.abort()
except BaseException:
logger.exception(
"failed to abort partially initialized nccl_m2n communicator"
)
if handle is not None:
try:
handle.destroy()
except BaseException:
logger.exception(
"failed to destroy partially initialized nccl_m2n handle"
)
raise
self._handle = handle
self.model_update_group = group
def _prepare_source_plan(
self,
wire_params: tuple[M2NWireParam, ...],
num_trainer_ranks: int,
) -> None:
"""Validate worker-visible source metadata without constructing NCCL."""
self._metas = []
for param in wire_params:
dtype = getattr(torch, param.dtype_name, None)
if not isinstance(dtype, torch.dtype):
raise ValueError(
f"parameter '{param.name}' has invalid dtype name "
f"'{param.dtype_name}'; expected the name of a torch.dtype"
)
check_transferable(param.name, dtype, param.shape)
layout = M2NLayout(M2NMesh(param.src_mesh_dims, 0), param.src_placements)
if layout.mesh.size != num_trainer_ranks:
raise ValueError(
f"parameter '{param.name}' source mesh covers "
f"{layout.mesh.size} ranks, but there are "
f"{num_trainer_ranks} trainer ranks"
)
meta = M2NParamMeta(param.name, dtype, param.shape, layout)
expected_local_shape(meta)
self._metas.append(meta)
def _prepare_destination_plan(self, init_info: M2NWeightTransferInitInfo) -> None:
"""Resolve the merged-#51520 in-place/full-fallback destination plan."""
# The inference topology, as declared by the trainer. This is the mesh
# a sharded destination is placed over; a replicated one is described
# by the alternate descriptor `resolve_layout` derives from it, which
# covers the same ranks but is not this mesh.
self._dst_mesh = M2NMesh(
cast(tuple[int, int], tuple(init_info.dst_mesh_dims)),
init_info.rank_offset,
)
parallel_config = self.parallel_config
shard_axis_size = self._dst_mesh.dims[DESTINATION_SHARD_AXIS]
quantization = getattr(self.model_config, "quantization", None)
# A quantized weight loader may dequantize / transpose / requantize, and
# under PP a rank's local shape is not the checkpoint shape split over
# the shard axis. Both break shape-derived resolution, so take the
# fallback.
allow_direct = (
quantization is None and parallel_config.pipeline_parallel_size == 1
)
# Match every incoming checkpoint parameter to its rank-local target.
# Resolvable parameters can be resharded directly into live model
# storage; the rest receive a full tensor for `load_weights`.
self._parameter_destinations = resolve_parameter_destinations(
self.model,
[m.name for m in self._metas],
[m.dtype for m in self._metas],
[m.shape for m in self._metas],
num_workers=self._dst_mesh.size,
shard_axis_size=shard_axis_size,
allow_direct=allow_direct,
)
validate_transfer_plan(
self._metas,
self._dst_mesh,
[destination.placements for destination in self._parameter_destinations],
)
# Update requests carry names only, so cache their plan indices. Track
# whether any fallback entry requires the `load_weights` lifecycle.
self._index = {meta.name: i for i, meta in enumerate(self._metas)}
self._uses_load_weights = any(
not destination.direct for destination in self._parameter_destinations
)
def start_weight_update(self) -> None:
"""Set up layerwise reloading, but only if some parameter needs it.
Directly-resharded parameters are written in place and never go through
`load_weights`, so a plan with no fallback entries has nothing to
reload.
"""
if not self._uses_load_weights:
return
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
initialize_layerwise_reload(self.model)
def finish_weight_update(self) -> None:
"""Finalize layerwise reloading when the plan uses fallback entries."""
if not self._uses_load_weights:
return
from vllm.model_executor.model_loader.reload import finalize_layerwise_reload
finalize_layerwise_reload(self.model, self.model_config)
def receive_weights(self, update_info: M2NWeightTransferUpdateInfo) -> None:
"""Receive each requested parameter using its initialization-time plan."""
if self._handle is None or self.model_update_group is None:
raise RuntimeError(
"nccl_m2n weight transfer not initialized. "
"Call init_transfer_engine() first."
)
if not isinstance(update_info.names, list) or any(
not isinstance(name, str) for name in update_info.names
):
raise TypeError("nccl_m2n update names must be a list of strings")
requested = []
for name in update_info.names:
index = self._index.get(name)
if index is None:
raise ValueError(
f"parameter '{name}' was not declared at init; the "
"trainer must send the same parameter set it announced"
)
requested.append((name, index))
from vllm.model_executor.model_loader.mtp_validation import (
disable_mtp_completeness_check,
)
stream = torch.cuda.current_stream()
# Reshard lazily so one `load_weights` invocation sees the complete
# fallback sequence without staging every full tensor at once. Direct
# destinations still execute in request order as the iterator advances.
def reshard_requested_weights() -> Iterator[tuple[str, torch.Tensor]]:
for name, index in requested:
meta = self._metas[index]
destination = self._parameter_destinations[index]
buffer = destination.tensor
if buffer is None:
buffer = torch.empty(
meta.shape, dtype=meta.dtype, device=self.device
)
self._reshard(
self.model_update_group,
stream,
meta,
destination.placements,
buffer,
)
if destination.tensor is None:
# `load_weights` reads on the host stream, so the transfer
# has to have landed before it runs.
stream.synchronize()
yield name, buffer
has_fallback = any(
self._parameter_destinations[index].tensor is None for _, index in requested
)
with disable_mtp_completeness_check():
received_weights = reshard_requested_weights()
if has_fallback:
# Some model loaders keep invocation-local state to combine
# related checkpoint tensors, so preserve one iterable scope.
self.model.load_weights(received_weights)
else:
# The iterator performs the reshards. With nothing to yield to
# `load_weights`, exhaust it here to execute direct transfers.
for _ in received_weights:
pass
def _reshard(
self,
comm: Any,
stream: torch.cuda.Stream,
src_meta: M2NParamMeta,
dst_placements: Placements | None,
dst_buffer: torch.Tensor,
) -> None:
"""Reshard one parameter from its trainer layout into a worker buffer.
`dst_placements` describes the buffer's logical placement over the
worker mesh; `dst_buffer` is this rank's physical storage.
"""
assert self._dst_mesh is not None
m2n = self._m2n
src_mesh, resolved_src_placements = resolve_layout(
src_meta.source_layout.mesh, src_meta.source_layout.placements
)
dst_mesh, resolved_dst_placements = resolve_layout(
self._dst_mesh, dst_placements
)
m2n.reshard(
None,
dst_buffer,
comm,
stream,
src_mesh=to_mesh(m2n, src_mesh),
src_placements=to_placements(m2n, resolved_src_placements),
dst_mesh=to_mesh(m2n, dst_mesh),
dst_placements=to_placements(m2n, resolved_dst_placements),
handle=self._handle,
)
def shutdown(self) -> None:
"""Finish pending GPU work and release M2N communication state."""
handle, self._handle = self._handle, None
group, self.model_update_group = self.model_update_group, None
self._m2n = None
self._metas = []
self._dst_mesh = None
self._parameter_destinations = []
self._index = {}
self._uses_load_weights = False
first_error: BaseException | None = None
def attempt(action: Any, message: str) -> None:
nonlocal first_error
try:
action()
except BaseException as exc:
if first_error is None:
first_error = exc
logger.exception(message)
if handle is not None or group is not None:
# M2N does not synchronize caller streams on finalize.
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
if handle is not None:
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
if group is not None:
attempt(group.abort, "failed to abort nccl_m2n communicator")
if first_error is not None:
raise first_error
@@ -1,261 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Destination-layout resolution for the NCCL M2N weight transfer backend.
For every incoming checkpoint parameter the worker needs a destination buffer
plus its placement over the inference mesh. Two outcomes are possible:
* **direct** — when the model is not quantized, the parameter maps 1:1 onto a
live vLLM parameter and its loader is a small, known-safe copy or tensor-
parallel loader. The loader's declared input/output dimension determines the
placement; shapes are validation, not an inference mechanism. The reshard
writes straight into the live parameter, so each rank receives only its own
shard and nothing is copied afterwards.
* **fallback** — anything else. The reshard delivers the whole tensor to every
rank and `load_weights` does the sharding, exactly as the broadcast NCCL
backend does. Fused parameters (`qkv_proj`, `gate_up_proj`, MoE `w13`/`w2`)
take this path: the checkpoint name does not name a vLLM parameter, so there
is nothing to resolve against.
Correctness never depends on a parameter resolving — the fallback is always
available and is the same path the existing backend uses.
"""
import inspect
import math
from collections.abc import Sequence
from dataclasses import dataclass
from functools import partial
from typing import cast
import torch
from vllm.distributed.weight_transfer.m2n_common import (
DESTINATION_REPLICA_AXIS,
DESTINATION_SHARD_AXIS,
MESH_NDIMS,
REPLICATE,
REPLICATED,
Placements,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
_DIRECT_WEIGHT_LOADING_BLOCKLIST = {
"vllm.model_executor.models.gpt2.GPT2Model",
}
def _module_type_name(module: torch.nn.Module) -> str:
cls = type(module)
return f"{cls.__module__}.{cls.__name__}"
def _destination_placements(shard_dim: int) -> Placements:
"""Place one tensor shard on the configured destination mesh axis."""
placements = [REPLICATE] * MESH_NDIMS
placements[DESTINATION_REPLICA_AXIS] = REPLICATE
placements[DESTINATION_SHARD_AXIS] = shard_dim
return cast(Placements, tuple(placements))
@dataclass
class M2NDestination:
"""Where one checkpoint parameter lands on this worker."""
name: str
placements: Placements | None
"""Placement over the inference mesh, or `REPLICATED` for the fallback."""
tensor: torch.Tensor | None
"""The live parameter view to reshard into, or None for the fallback path
(the engine allocates a full replica per round and calls `load_weights`)."""
@property
def direct(self) -> bool:
return self.tensor is not None
def _base_loader_shard_dim(
param: torch.nn.Parameter,
shard_axis_size: int,
) -> int | None:
"""Return a known-safe loader's declared shard dim, or reject it.
Loader identity is deliberately fail-closed. A missing loader is not
treated as ``default_weight_loader`` because a model-level ``load_weights``
implementation may transform the tensor before it reaches that default.
Wrappers and partials are also rejected: recognizing their wrapped callable
would ignore behavior added by the wrapper.
"""
loader = getattr(param, "weight_loader", None)
if loader is None or isinstance(loader, partial):
return None
owner = loader.__self__ if inspect.ismethod(loader) else None
loader_fn = loader.__func__ if inspect.ismethod(loader) else loader
# Lazy imports avoid making the weight-transfer registry import all model
# executor layers during normal vLLM startup.
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
replicated_loaders = {
default_weight_loader,
ReplicatedLinear.weight_loader,
}
column_loaders = {
ColumnParallelLinear.weight_loader,
ColumnParallelLinear.weight_loader_v2,
}
row_loaders = {
RowParallelLinear.weight_loader,
RowParallelLinear.weight_loader_v2,
}
# These attributes signal packing, a transform, or pre-sharded checkpoint
# storage. Such semantics cannot be reproduced by a plain M2N placement.
# Check them before accepting replicated loaders too: model-level loading
# may transform a tensor before handing it to an otherwise plain copier.
metadata_attrs = ("packed_dim", "packed_factor", "marlin_tile_size")
flag_attrs = ("needs_scalar_to_array", "is_sharded_weight", "is_transposed")
if any(getattr(param, attr, None) is not None for attr in metadata_attrs):
return None
if any(bool(getattr(param, attr, False)) for attr in flag_attrs):
return None
if loader_fn in replicated_loaders:
return REPLICATE
if loader_fn not in column_loaders | row_loaders:
return None
if owner is None or getattr(owner, "tp_size", None) != shard_axis_size:
return None
attr = "output_dim" if loader_fn in column_loaders else "input_dim"
dim = getattr(param, attr, None)
return dim if isinstance(dim, int) and dim >= 0 else None
def _validated_shard_dim(
global_shape: Sequence[int],
local_shape: Sequence[int],
declared_dim: int,
shard_axis_size: int,
) -> int | None:
"""Validate a loader-declared shard dimension against both shapes."""
if len(global_shape) != len(local_shape):
return None
if declared_dim == REPLICATE:
return REPLICATE if tuple(global_shape) == tuple(local_shape) else None
if declared_dim >= len(global_shape):
return None
for dim, (whole, local) in enumerate(zip(global_shape, local_shape)):
expected = local * shard_axis_size if dim == declared_dim else local
if whole != expected:
return None
return declared_dim
def resolve_parameter_destinations(
model: torch.nn.Module,
names: Sequence[str],
dtypes: Sequence[torch.dtype],
shapes: Sequence[Sequence[int]],
*,
num_workers: int,
shard_axis_size: int,
allow_direct: bool,
) -> list[M2NDestination]:
"""Build this worker's destination plan, one entry per checkpoint parameter.
Placements are relative to the inference mesh: axis 0 replicates and axis 1
shards. `allow_direct=False` forces every parameter onto the fallback path;
the engine sets it for pipeline-parallel or quantized deployments, where a
parameter's local shape is not simply the checkpoint shape split across the
shard axis. Parameters below a model type with known model-level checkpoint
preprocessing also fall back, since bypassing `load_weights` would skip the
transformation.
"""
params = dict(model.named_parameters()) if allow_direct else {}
blocked_param_ids = {
id(param)
for module in model.modules()
if _module_type_name(module) in _DIRECT_WEIGHT_LOADING_BLOCKLIST
for param in module.parameters()
}
destinations: list[M2NDestination] = []
for name, dtype, shape in zip(names, dtypes, shapes):
param = params.get(name)
dim = None
if (
param is not None
and id(param) not in blocked_param_ids
and param.dtype == dtype
and param.data.is_contiguous()
and num_workers % shard_axis_size == 0
):
declared_dim = _base_loader_shard_dim(param, shard_axis_size)
if declared_dim is not None:
dim = _validated_shard_dim(
shape, param.shape, declared_dim, shard_axis_size
)
if dim is None:
# A replicated parameter is identical on every rank, so it needs no
# placement of its own — REPLICATED lets resolve_layout spread it
# over all the workers regardless of how the mesh is factored.
destinations.append(M2NDestination(name, REPLICATED, None))
elif dim == REPLICATE:
assert param is not None
destinations.append(M2NDestination(name, REPLICATED, param.data))
else:
assert param is not None
destinations.append(
M2NDestination(name, _destination_placements(dim), param.data)
)
# Layerwise reload finalizes a module as a unit. Mixing direct and fallback
# parameters within one module can copy stale storage over the direct load.
fallback_modules = {
name.rpartition(".")[0]
for name, destination in zip(names, destinations)
if not destination.direct and name in params
}
destinations = [
M2NDestination(destination.name, REPLICATED, None)
if destination.direct
and destination.name.rpartition(".")[0] in fallback_modules
else destination
for destination in destinations
]
num_direct = sum(d.direct for d in destinations)
parameter_bytes = [
math.prod(shape) * dtype.itemsize for dtype, shape in zip(dtypes, shapes)
]
total_bytes = sum(parameter_bytes)
direct_bytes = sum(
nbytes
for destination, nbytes in zip(destinations, parameter_bytes)
if destination.direct
)
direct_byte_percentage = 100 * direct_bytes / total_bytes if total_bytes else 0
logger.info(
"nccl_m2n destination plan: %d/%d parameters resharded directly into "
"the model, %d via full-tensor fallback; direct byte coverage: "
"%d/%d bytes (%.1f%%)",
num_direct,
len(destinations),
len(destinations) - num_direct,
direct_bytes,
total_bytes,
direct_byte_percentage,
)
return destinations
@@ -1,207 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Trainer-side weight sources for the NCCL M2N backend.
m2n plans a transfer from both sides' layouts, so the trainer has to say how it
holds each parameter — which the base `WeightSource` / `ParamMeta` pair does not
express. This module supplies the m2n flavor of both, and the DTensor-backed
implementation that covers the common trainer. Each parameter owns its source
mesh and placements because dense and expert weights can use different
factorizations over the same trainer ranks.
"""
from collections.abc import Callable, Iterator, Sequence
from dataclasses import dataclass
from typing import Any
import torch
from vllm.distributed.weight_transfer.base import WeightSource
from vllm.distributed.weight_transfer.m2n_common import (
REPLICATE,
REPLICATED,
M2NLayout,
M2NMesh,
M2NParamMeta,
Placements,
)
__all__ = [
"M2NWeightSource",
"M2NManifestEntry",
"ManifestM2NWeightSource",
"DTensorModuleSource",
"mesh_from_tensor",
"placements_from_tensor",
]
class M2NWeightSource(WeightSource):
"""A `WeightSource` that also describes how the trainer holds its weights.
Unlike `ModuleSource`, iteration yields each rank's **local shard**, not a
materialized full tensor: gathering is exactly the cost m2n removes.
"""
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
"""Name, dtype, full shape, and source layout for each parameter."""
raise NotImplementedError
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
"""Yield `(name, local buffer)` pairs in metadata order.
Providers must enqueue materialization on the caller's current CUDA
stream, or return a buffer whose writes are already visible to it.
Buffers may be torch tensors, CUDA Array Interface objects, or raw
device pointers accepted by nccl4py. Providers own correct
communicator-device placement. Raw-pointer providers additionally own
allocation lifetime, dtype, and shape correctness.
"""
raise NotImplementedError
@dataclass(frozen=True)
class M2NManifestEntry:
"""One immutable source descriptor with a lazy rank-local tensor view."""
key: str
dtype: torch.dtype
global_shape: tuple[int, ...]
source_layout: M2NLayout
local_tensor: Callable[[], Any]
def __post_init__(self) -> None:
object.__setattr__(self, "global_shape", tuple(self.global_shape))
if not self.key:
raise ValueError("M2N manifest key must not be empty")
if not isinstance(self.dtype, torch.dtype):
raise TypeError(
f"manifest entry '{self.key}' has invalid dtype {self.dtype!r}"
)
if not isinstance(self.source_layout, M2NLayout):
raise TypeError(
f"manifest entry '{self.key}' source_layout must be an M2NLayout"
)
if not callable(self.local_tensor):
raise TypeError(
f"manifest entry '{self.key}' local_tensor must be callable"
)
@property
def metadata(self) -> M2NParamMeta:
"""Return the serializable portion consumed during initialization."""
return M2NParamMeta(
self.key,
self.dtype,
self.global_shape,
self.source_layout,
)
class ManifestM2NWeightSource(M2NWeightSource):
"""An ordered M2N source backed by lazy, rank-local manifest entries."""
def __init__(self, entries: Sequence[M2NManifestEntry]) -> None:
self._entries = tuple(entries)
self._metadata = tuple(entry.metadata for entry in self._entries)
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
"""Return a fresh list while preserving the immutable manifest order."""
return list(self._metadata)
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
"""Materialize one rank-local buffer at a time."""
for entry in self._entries:
yield entry.key, entry.local_tensor()
def _placement_code(placement: Any) -> int:
"""Map a `torch.distributed` placement onto an m2n placement code."""
name = type(placement).__name__
if name == "Replicate":
return REPLICATE
if name == "Shard":
return int(placement.dim)
raise ValueError(
f"nccl_m2n cannot express the {name} placement; only Replicate and "
"Shard are supported"
)
def mesh_from_tensor(tensor: torch.Tensor, num_trainer_ranks: int) -> M2NMesh:
"""The mesh a parameter lives on, as an `M2NMesh`.
Tensors without DeviceMesh metadata are treated as replicated. Implicitly
sharded tensors require a custom source with an explicit layout.
"""
device_mesh = getattr(tensor, "device_mesh", None)
if device_mesh is None:
return M2NMesh((num_trainer_ranks, 1), 0)
grid = device_mesh.mesh
ranks = grid.flatten().tolist()
if ranks != list(range(len(ranks))):
raise ValueError(
"nccl_m2n requires the trainer's device mesh to cover the "
f"contiguous rank interval [0, {len(ranks)}); got {ranks}"
)
if grid.ndim > 2:
raise ValueError(
f"nccl_m2n supports 1-D and 2-D device meshes, got {grid.ndim}-D"
)
dims = tuple(grid.shape)
return M2NMesh(dims if len(dims) == 2 else (dims[0], 1), 0)
def placements_from_tensor(tensor: torch.Tensor) -> Placements | None:
"""How a parameter is placed over its mesh, or `REPLICATED`.
`REPLICATED` is returned rather than a `(REPLICATE, REPLICATE)` pair: m2n
cannot express that, and the size-1-axis encoding it needs instead is
applied later by `resolve_layout`, which owns that workaround.
"""
placements = getattr(tensor, "placements", None)
if placements is None:
return REPLICATED
codes = [_placement_code(p) for p in placements]
if all(code == REPLICATE for code in codes):
return REPLICATED
if len(codes) == 1:
return (codes[0], REPLICATE)
return (codes[0], codes[1])
class DTensorModuleSource(M2NWeightSource):
"""`M2NWeightSource` over `module.named_parameters()`.
Covers both the FSDP/DTensor trainer (placement read off each parameter,
local shard yielded via `to_local()`) and the plain replicated trainer, with
no special casing. Trainers with a custom producer (a Megatron export, MoE
re-fusing) subclass `M2NWeightSource` instead.
"""
def __init__(self, module: torch.nn.Module, num_trainer_ranks: int) -> None:
"""Wrap `module`; `num_trainer_ranks` sizes the mesh for plain tensors."""
self._module = module
self._num_trainer_ranks = num_trainer_ranks
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
"""Read each global shape and source layout without gathering shards."""
return [
M2NParamMeta(
name,
p.dtype,
tuple(p.shape),
M2NLayout(
mesh_from_tensor(p, self._num_trainer_ranks),
placements_from_tensor(p),
),
)
for name, p in self._module.named_parameters()
]
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
"""Yield each parameter's local shard (`to_local()`), or the tensor itself."""
for name, param in self._module.named_parameters():
to_local = getattr(param, "to_local", None)
yield name, (to_local() if callable(to_local) else param)
@@ -1,628 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Trainer-side weight transfer engine for the NCCL M2N backend.
Symmetric to `M2NWeightTransferEngine` but in the training process. Every
trainer rank joins the shared communicator and runs every reshard, sending its
own local shard; only rank 0 touches the inference control plane.
Workers join the communicator during init and run reshard from inside
`update_weights`, so those two RPCs overlap with work on this side. Both run on
a helper thread and are joined afterwards, which keeps that overlap inside the
engine -- where the `TrainerWeightTransferEngine` contract puts it -- rather than
in every caller.
"""
import socket
from collections.abc import Mapping
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar, Self, cast
import torch
from vllm.distributed.weight_transfer.base import (
TrainerInitInfo,
TrainerWeightTransferEngine,
VLLMWeightSyncClient,
WeightSource,
)
from vllm.distributed.weight_transfer.m2n_common import (
M2N_WIRE_SCHEMA_VERSION,
M2NMesh,
M2NParamMeta,
M2NWireParam,
Placements,
check_complete_plan_agreement,
check_data_plane_agreement,
check_runtime_ready_agreement,
check_source_plan_agreement,
check_transferable,
create_m2n_communicator,
expected_local_shape,
import_m2n,
prepare_m2n_local_runtime,
publish_destination_placements,
resolve_layout,
source_plan_digest,
to_mesh,
to_placements,
validate_local_buffer,
validate_transfer_plan,
)
from vllm.distributed.weight_transfer.m2n_source import M2NWeightSource
from vllm.distributed.weight_transfer.nccl_common import (
decode_nccl_unique_id,
stateless_init_metadata_group,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.distributed.utils import StatelessProcessGroup
logger = init_logger(__name__)
__all__ = ["M2NTrainerInitInfo", "M2NTrainerWeightTransferEngine"]
@dataclass(kw_only=True)
class M2NTrainerInitInfo(TrainerInitInfo):
"""Trainer-side init info for nccl_m2n.
`rank` (from `TrainerInitInfo`) is this trainer process's rank; it is also
its rank in the shared communicator, since the trainer occupies
`[0, num_trainer_ranks)`. Rank 0 drives the control plane.
"""
backend: ClassVar[str] = "nccl_m2n"
master_address: str
master_port: int
world_size: int
"""Trainer ranks + all inference workers."""
num_trainer_ranks: int = 1
dst_mesh_dims: tuple[int, int] | None = None
"""How the inference ranks are laid out: axis 0 replicates and axis 1
shards. The trainer declares it so both sides describe the destination
identically; defaults to a flat `(num_workers, 1)`, which is all a
replicated destination needs.
Destination meshes remain common to every parameter; per-parameter
destination meshes are intentionally future work.
"""
nccl_unique_id_b64: str | None = field(default=None, repr=False)
"""Optional UID for the NCCL data plane; TCP remains the metadata plane."""
max_cta: int | None = None
listen_socket: socket.socket | None = field(default=None, repr=False, compare=False)
"""Optional rank-zero TCPStore reservation; never sent to workers."""
@property
def destination_mesh_dims(self) -> tuple[int, int]:
"""`dst_mesh_dims`, or a flat mesh over every inference worker."""
if self.dst_mesh_dims is not None:
return cast(tuple[int, int], tuple(self.dst_mesh_dims))
return (self.world_size - self.num_trainer_ranks, 1)
def __post_init__(self) -> None:
"""Reject rank counts or destination meshes that cannot form a group."""
if not isinstance(self.master_address, str) or not self.master_address:
raise ValueError("`master_address` must name the metadata rendezvous")
if (
not isinstance(self.master_port, int)
or isinstance(self.master_port, bool)
or not 0 < self.master_port < 65536
):
raise ValueError("`master_port` must be a valid metadata port")
if not 0 < self.num_trainer_ranks < self.world_size:
raise ValueError(
f"`num_trainer_ranks` ({self.num_trainer_ranks}) must leave at "
f"least one worker in world_size {self.world_size}"
)
num_workers = self.world_size - self.num_trainer_ranks
if self.dst_mesh_dims is not None:
dims = tuple(self.dst_mesh_dims)
try:
mesh = M2NMesh(cast(tuple[int, int], dims), self.num_trainer_ranks)
except (TypeError, ValueError) as exc:
raise ValueError(f"invalid `dst_mesh_dims` {dims}: {exc}") from exc
if mesh.size != num_workers:
raise ValueError(
f"`dst_mesh_dims` {dims} must be two dims covering the "
f"{num_workers} inference workers"
)
if not 0 <= self.rank < self.num_trainer_ranks:
raise ValueError(
f"trainer `rank` ({self.rank}) must be below "
f"`num_trainer_ranks` ({self.num_trainer_ranks})"
)
if self.listen_socket is not None:
if self.rank != 0:
raise ValueError("only trainer rank zero may reserve the TCPStore")
if not isinstance(self.listen_socket, socket.socket):
raise TypeError("listen_socket must be a socket.socket")
if self.listen_socket.fileno() < 0:
raise ValueError("listen_socket is already closed")
@property
def nccl_unique_id_bytes(self) -> bytes | None:
if self.nccl_unique_id_b64 is None:
return None
return decode_nccl_unique_id(
master_address=None,
master_port=None,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="M2NTrainerInitInfo data plane",
)
class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitInfo]):
"""Trainer-side engine: sends every rank's local shard, once per parameter.
Called on every trainer rank. All ranks join the communicator and run every
reshard; only rank 0 touches the control plane.
"""
init_info_cls = M2NTrainerInitInfo
def __init__(
self,
*,
client: VLLMWeightSyncClient,
source: M2NWeightSource,
is_sender: bool,
) -> None:
"""Hold the client and source; `trainer_init` fills in the group."""
super().__init__(client=client, source=source, is_sender=is_sender)
self.group: Any = None
self._m2n: Any = None
self._handle: Any = None
self._metas: list[M2NParamMeta] = []
self._wire_params: list[M2NWireParam] = []
self._source_digest = ""
self._dst_mesh: M2NMesh | None = None
self._dst_placements: list[Placements | None] = []
self._metadata_group: StatelessProcessGroup | None = None
self._num_trainer_ranks = 0
@classmethod
def trainer_init(
cls,
init_info: M2NTrainerInitInfo,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
) -> Self:
"""Build the engine and rendezvous with the inference side.
Runs on *every* trainer rank. Rank 0 additionally ships the transfer
plan to the workers and drives the control plane; the other ranks only
build local state and join the communicator. Every rank participates
in every reshard and sends its local shard.
Build a local validation envelope, start the worker RPC without
waiting, join the metadata rendezvous, and require unanimous source,
data-plane, and runtime readiness before constructing NCCL.
"""
engine = cls(
client=client,
source=cast(M2NWeightSource, source),
is_sender=init_info.is_sender,
)
engine._initialize(init_info, source)
return engine
def _initialize(
self,
init_info: M2NTrainerInitInfo,
source: WeightSource | None,
) -> None:
"""Run source validation, metadata agreement, and NCCL rendezvous."""
self._num_trainer_ranks = init_info.num_trainer_ranks
local_source_error: str | None = None
try:
if not isinstance(source, M2NWeightSource):
raise TypeError(
"nccl_m2n needs per-parameter layouts, so its source must "
f"be an M2NWeightSource; got {type(source).__name__}"
)
self._m2n = import_m2n()
self._prepare_source_plan(source, init_info.num_trainer_ranks)
except Exception as exc:
local_source_error = f"{type(exc).__name__}: {exc}"
self._metas = []
self._wire_params = []
self._source_digest = source_plan_digest([])
executor: ThreadPoolExecutor | None = None
init_future: Future[None] | None = None
if self.is_sender:
executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="m2n-weight-sync"
)
init_future = executor.submit(
self.client.init_weight_transfer_engine,
self._worker_init_info(init_info),
)
handle = None
group = None
try:
reservation = init_info.listen_socket
try:
metadata_group = stateless_init_metadata_group(
init_info.master_address,
init_info.master_port,
init_info.rank,
init_info.world_size,
listen_socket=reservation,
)
finally:
init_info.listen_socket = None
if reservation is not None:
reservation.close()
plan_digest = check_source_plan_agreement(
metadata_group,
self._wire_params,
expected_digest=self._source_digest,
local_error=local_source_error,
)
# Retain the common destination mesh and the merged NCCL publication
# of per-parameter placements.
self._dst_mesh = M2NMesh(
init_info.destination_mesh_dims, init_info.num_trainer_ranks
)
unique_id_bytes: bytes | None = None
data_plane_error: str | None = None
try:
unique_id_bytes = init_info.nccl_unique_id_bytes
except Exception as exc:
data_plane_error = f"{type(exc).__name__}: {exc}"
check_data_plane_agreement(
metadata_group,
unique_id_bytes,
data_plane_error,
)
runtime_error: str | None = None
nccl = None
device = None
try:
nccl, device, handle = prepare_m2n_local_runtime(
self._m2n, init_info.max_cta
)
except Exception as exc:
runtime_error = f"{type(exc).__name__}: {exc}"
check_runtime_ready_agreement(metadata_group, runtime_error)
assert nccl is not None and device is not None and handle is not None
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
destination_placements = self._receive_destination_plan(
group,
init_info.num_trainer_ranks,
torch.device("cuda", device),
)
complete_plan_error: str | None = None
try:
assert self._dst_mesh is not None
validate_transfer_plan(
self._metas, self._dst_mesh, destination_placements
)
except Exception as exc:
complete_plan_error = f"{type(exc).__name__}: {exc}"
assert self._dst_mesh is not None
check_complete_plan_agreement(
metadata_group,
plan_digest,
self._dst_mesh,
destination_placements,
complete_plan_error,
)
if init_future is not None:
init_future.result()
logger.info(
"nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), "
"inference ranks [%d, %d)",
len(self._metas),
init_info.num_trainer_ranks,
init_info.num_trainer_ranks,
init_info.world_size,
)
logger.info("nccl_m2n source plan digest: %s", plan_digest)
except BaseException:
if group is not None:
try:
# Transport cleanup only; reload validity belongs to the
# model-update lifecycle, not this backend.
group.abort()
except BaseException:
logger.exception(
"failed to abort partially initialized nccl_m2n communicator"
)
if handle is not None:
try:
handle.destroy()
except BaseException:
logger.exception(
"failed to destroy partially initialized nccl_m2n handle"
)
raise
finally:
if executor is not None:
executor.shutdown(wait=False, cancel_futures=True)
self._metadata_group = metadata_group
self._handle = handle
self.group = group
self._dst_placements = destination_placements
def _receive_destination_plan(
self, group: Any, first_worker_rank: int, device: torch.device
) -> list[Placements | None]:
"""Receive the inference-side per-parameter placements."""
# The first inference worker publishes one placement per parameter.
# Every trainer rank receives the complete plan in this single,
# initialization-time broadcast.
placements = publish_destination_placements(
group,
device,
torch.cuda.current_stream(),
first_worker_rank,
None,
len(self._metas),
)
return placements
def _worker_init_info(self, init_info: M2NTrainerInitInfo) -> dict[str, Any]:
"""Handshake payload: rendezvous, both meshes, and the transfer plan."""
payload: dict[str, Any] = {
"schema_version": M2N_WIRE_SCHEMA_VERSION,
"master_address": init_info.master_address,
"master_port": init_info.master_port,
"nccl_unique_id_b64": init_info.nccl_unique_id_b64,
"rank_offset": init_info.num_trainer_ranks,
"world_size": init_info.world_size,
"dst_mesh_dims": list(init_info.destination_mesh_dims),
"source_digest": self._source_digest,
"params": [param.to_dict() for param in self._wire_params],
"max_cta": init_info.max_cta,
}
return {key: value for key, value in payload.items() if value is not None}
def _prepare_source_plan(
self, source: M2NWeightSource, num_trainer_ranks: int
) -> None:
"""Build and validate this rank's immutable source plan."""
self._metas = source.metadata()
names: set[str] = set()
for meta in self._metas:
if not isinstance(meta, M2NParamMeta):
raise TypeError(
"nccl_m2n source metadata must contain M2NParamMeta "
f"entries, got {type(meta).__name__}"
)
if meta.name in names:
raise ValueError(
"nccl_m2n source metadata contains duplicate parameter "
f"'{meta.name}'"
)
names.add(meta.name)
check_transferable(meta.name, meta.dtype, meta.shape)
layout = meta.source_layout
if layout.mesh.start_rank != 0:
raise ValueError(
f"parameter '{meta.name}' source mesh must start at rank 0, "
f"got {layout.mesh.start_rank}"
)
if layout.mesh.size != num_trainer_ranks:
raise ValueError(
f"parameter '{meta.name}' source mesh covers "
f"{layout.mesh.size} ranks, but there are "
f"{num_trainer_ranks} trainer ranks"
)
expected_local_shape(meta)
self._wire_params = [
M2NWireParam(
name=meta.name,
dtype_name=str(meta.dtype).split(".")[-1],
shape=meta.shape,
src_mesh_dims=meta.source_layout.mesh.dims,
src_placements=meta.source_layout.placements,
)
for meta in self._metas
]
self._source_digest = source_plan_digest(self._wire_params)
def send_weights(self) -> None:
"""Drive one update round: start, reshard concurrently, then finish."""
if self._handle is None or self.group is None:
raise RuntimeError("nccl_m2n trainer engine is not initialized")
if not self.is_sender:
try:
self._send()
except BaseException:
try:
self._receive_round_outcome()
except BaseException:
logger.exception(
"failed to consume nccl_m2n round outcome after local failure"
)
raise
self._receive_round_outcome()
return
try:
self.client.start_weight_update()
executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="m2n-weight-sync"
)
try:
update = executor.submit(
self.client.update_weights,
{"names": [m.name for m in self._metas]},
)
# Surface an immediate worker-side rejection before trainers
# enter the collective sequence.
if update.done():
update.result()
self._send()
update.result()
finally:
executor.shutdown(wait=False, cancel_futures=True)
self.client.finish_weight_update()
except BaseException as exc:
try:
self._publish_round_outcome(exc)
except BaseException:
logger.exception("failed to publish nccl_m2n round failure")
raise
self._publish_round_outcome(None)
def _publish_round_outcome(self, failure: BaseException | None) -> None:
"""Tell non-sender trainer ranks whether worker finalization passed."""
if self._num_trainer_ranks <= 1:
return
if self._metadata_group is None:
raise RuntimeError("nccl_m2n metadata group is unavailable")
envelope = {
"ok": failure is None,
"error": (
None if failure is None else f"{type(failure).__name__}: {failure}"
),
}
for rank in range(1, self._num_trainer_ranks):
self._metadata_group.send_obj(envelope, rank)
# Keep rank zero's TCPStore alive until every trainer consumed its
# outcome. This also keeps the per-rank message counters aligned.
expected_ack = {"phase": "round_outcome_consumed"}
for rank in range(1, self._num_trainer_ranks):
ack = self._metadata_group.recv_obj(rank)
if ack != expected_ack:
raise RuntimeError(
f"trainer rank {rank} sent an invalid round-outcome acknowledgement"
)
def _receive_round_outcome(self) -> None:
"""Wait until trainer rank zero observed every worker's finish RPC."""
if self._metadata_group is None:
raise RuntimeError("nccl_m2n metadata group is unavailable")
outcome = self._metadata_group.recv_obj(0)
self._metadata_group.send_obj({"phase": "round_outcome_consumed"}, 0)
if not isinstance(outcome, Mapping) or set(outcome) != {"ok", "error"}:
raise RuntimeError("trainer rank zero sent an invalid round outcome")
ok = outcome["ok"]
error = outcome["error"]
if not isinstance(ok, bool) or ok != (error is None):
raise RuntimeError("trainer rank zero sent an invalid round outcome")
if not ok:
if not isinstance(error, str) or not error:
raise RuntimeError("trainer rank zero sent an invalid round error")
raise RuntimeError(
"nccl_m2n worker update failed on trainer rank zero: " + error
)
def _send(self) -> None:
"""Reshard each local shard into its worker-planned destination."""
assert self.group is not None
assert self._dst_mesh is not None
assert self.source is not None
m2n = self._m2n
stream = torch.cuda.current_stream()
device_index = torch.accelerator.current_device_index()
# Optimizer work may have used another CUDA stream. Make prior writes
# visible before a provider materializes current-stream local shards.
torch.accelerator.synchronize(device_index)
inflight_buffers: list[Any] = []
sent = 0
for index, (name, buffer) in enumerate(self.source):
if index >= len(self._metas):
raise RuntimeError(
"weight source yielded more parameters than metadata() "
f"declared (first extra parameter: '{name}')"
)
meta = self._metas[index]
if name != meta.name:
raise RuntimeError(
f"weight source yielded '{name}' at position {index}, but "
f"its metadata declared '{meta.name}'; the source must "
"iterate in a stable order"
)
validate_local_buffer(m2n, meta, buffer)
src_mesh, src_placements = resolve_layout(
meta.source_layout.mesh, meta.source_layout.placements
)
dst_mesh, dst_placements = resolve_layout(
self._dst_mesh, self._dst_placements[index]
)
raw_pointer_args = {}
if isinstance(buffer, int):
raw_pointer_args = {
"src_local_shape": expected_local_shape(meta),
"src_dtype": meta.dtype,
}
m2n.reshard(
buffer,
None,
self.group,
stream,
src_mesh=to_mesh(m2n, src_mesh),
src_placements=to_placements(m2n, src_placements),
dst_mesh=to_mesh(m2n, dst_mesh),
dst_placements=to_placements(m2n, dst_placements),
handle=self._handle,
**raw_pointer_args,
)
# Let allocator-aware buffers recycle storage in stream order.
# Other object owners must stay alive until the final synchronize;
# raw-pointer lifetime remains the source's responsibility.
record_stream = getattr(buffer, "record_stream", None)
if callable(record_stream):
record_stream(stream)
elif not isinstance(buffer, int):
inflight_buffers.append(buffer)
sent += 1
if sent != len(self._metas):
missing = self._metas[sent].name
raise RuntimeError(
f"weight source yielded {sent} parameters, but metadata() "
f"declared {len(self._metas)} (first missing: '{missing}')"
)
# As in the other weight-transfer backends, wait before returning
# rather than relying on same-stream ordering. The caller may then
# safely mutate parameters or continue work on another stream.
stream.synchronize()
def shutdown(self) -> None:
"""Finish pending GPU work and release M2N communication state."""
handle, self._handle = self._handle, None
group, self.group = self.group, None
self._metadata_group = None
self._dst_placements = []
self._dst_mesh = None
self._m2n = None
first_error: BaseException | None = None
def attempt(action: Any, message: str) -> None:
nonlocal first_error
try:
action()
except BaseException as exc:
if first_error is None:
first_error = exc
logger.exception(message)
if handle is not None or group is not None:
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
if handle is not None:
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
if group is not None:
attempt(group.abort, "failed to abort nccl_m2n communicator")
if first_error is not None:
raise first_error
@@ -1,41 +0,0 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Re-export the weight-transfer backend maintained by ModelExpress.
Install ModelExpress from the main branch's modelexpress_client/python
subdirectory.
"""
try:
from modelexpress import configure_vllm_logging
from modelexpress_rl.inference.engines.vllm.weight_transfer_engine import (
ModelExpressWeightTransferEngine,
ModelExpressWeightTransferInitInfo,
ModelExpressWeightTransferUpdateInfo,
)
except ModuleNotFoundError as exc:
if exc.name not in {
"modelexpress",
"modelexpress_rl",
"modelexpress_rl.inference",
"modelexpress_rl.inference.engines",
"modelexpress_rl.inference.engines.vllm",
"modelexpress_rl.inference.engines.vllm.weight_transfer_engine",
}:
raise
raise ImportError(
"The 'modelexpress' weight transfer backend requires ModelExpress. "
"Install it with `uv pip install "
"'git+https://github.com/ai-dynamo/modelexpress@main"
"#subdirectory=modelexpress_client/python'`."
) from exc
configure_vllm_logging()
__all__ = [
"ModelExpressWeightTransferEngine",
"ModelExpressWeightTransferInitInfo",
"ModelExpressWeightTransferUpdateInfo",
]
@@ -1,299 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared NCCL initialization helpers for weight transfer engines.
The dense (`NCCLWeightTransferEngine`) and sparse
(`SparseNCCLWeightTransferEngine`) backends are independent engines that share
*only* their process-group initialization. That common logic lives here so the
sparse engine does not have to subclass the dense one.
"""
import socket
from dataclasses import asdict, dataclass, field
from typing import TYPE_CHECKING, Protocol
import pybase64 as base64
import torch
if TYPE_CHECKING:
from vllm.config.parallel import ParallelConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
from vllm.distributed.device_communicators.pynccl_wrapper import (
NCCL_UNIQUE_ID_BYTES,
)
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
DEFAULT_PACKED_NUM_BUFFERS,
)
from vllm.utils.nccl import unpinned_nccl_env
def decode_nccl_unique_id(
*,
master_address: str | None,
master_port: int | None,
nccl_unique_id_b64: str | None,
ctx: str,
) -> bytes | None:
"""Validate the rendezvous mode and decode a pre-shared unique id."""
has_uid = bool(nccl_unique_id_b64)
has_addr = master_address is not None
has_port = master_port is not None
if has_uid and (has_addr or has_port):
raise ValueError(
f"{ctx}: pass nccl_unique_id_b64 OR master_address/master_port, not both"
)
if not has_uid:
if not (has_addr and has_port):
raise ValueError(
f"{ctx}: need nccl_unique_id_b64, or both master_address and "
f"master_port (got master_address={master_address!r}, "
f"master_port={master_port!r})"
)
return None
try:
decoded = base64.b64decode(nccl_unique_id_b64, validate=True)
except (ValueError, TypeError) as e:
raise ValueError(
f"{ctx}: nccl_unique_id_b64 is not valid standard base64 "
f"(URL-safe base64 is not accepted)"
) from e
if len(decoded) != NCCL_UNIQUE_ID_BYTES:
raise ValueError(
f"{ctx}: nccl_unique_id_b64 must decode to {NCCL_UNIQUE_ID_BYTES} "
f"bytes, got {len(decoded)}"
)
return decoded
@dataclass(kw_only=True)
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
"""Worker-side initialization info for NCCL-based weight transfer backends.
Keyword-only (`kw_only`): adding the optional `nccl_unique_id_b64` field
means the rendezvous fields can no longer keep a fixed positional slot, so a
stale positional call fails loudly instead of silently swapping arguments.
Provide exactly one rendezvous mode:
* ``master_address`` + ``master_port`` -- TCPStore / ``StatelessProcessGroup``
rendezvous (requires torch on every rank, including the trainer), or
* ``nccl_unique_id_b64`` -- standard (RFC 4648, *not* URL-safe) base64 of the
128 raw bytes from ``ncclGetUniqueId``, for torch-free trainers (e.g. JAX)
that mint the unique id out of band and share it (over HTTP, etc.). Note a
JAX peer must use ``base64.b64encode``, not ``urlsafe_b64encode``.
On the unique-id path all ranks must enter init concurrently (there is no
store barrier), and every peer must honor the warm-up handshake: the
worker's communicator issues a one-element ``all_reduce`` immediately after
``ncclCommInitRank`` (see ``PyNcclCommunicator.from_unique_id_bytes``), so a
foreign peer must issue a matching one-element ``all_reduce`` before any
other collective or all ranks deadlock.
"""
master_address: str | None = None
master_port: int | None = None
rank_offset: int
world_size: int
nccl_unique_id_b64: str | None = field(default=None, repr=False)
packed: bool = False
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
def __post_init__(self) -> None:
_ = self.nccl_unique_id_bytes
@property
def nccl_unique_id_bytes(self) -> bytes | None:
return decode_nccl_unique_id(
master_address=self.master_address,
master_port=self.master_port,
nccl_unique_id_b64=self.nccl_unique_id_b64,
ctx="NCCLWeightTransferInitInfo",
)
def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
only the mode actually in use. Shared by the dense and sparse trainer
engines so the two cannot drift."""
return {key: value for key, value in asdict(init_info).items() if value is not None}
class NCCLRendezvous(Protocol):
"""The TCP rendezvous fields `trainer_init` needs.
Structural so each backend can keep its own trainer init info next to its
engine (dense `NCCLTrainerInitInfo`, `SparseNCCLTrainerInitInfo`) without
this shared module importing either.
"""
master_address: str
master_port: int
world_size: int
def stateless_init_metadata_group(
master_address: str,
master_port: int,
rank: int,
world_size: int,
*,
listen_socket: socket.socket | None = None,
) -> "StatelessProcessGroup":
"""Create only the TCPStore group used for metadata agreement."""
from vllm.distributed.utils import StatelessProcessGroup
return StatelessProcessGroup.create(
host=master_address,
port=master_port,
rank=rank,
world_size=world_size,
listen_socket=listen_socket,
)
def stateless_init_process_group(
master_address: str,
master_port: int,
rank: int,
world_size: int,
device,
) -> "PyNcclCommunicator":
"""VLLM provides `StatelessProcessGroup` to create a process group
without considering the global process group in torch.distributed.
It is recommended to create `StatelessProcessGroup`, and then initialize
the data-plane communication (NCCL) between external (train processes)
and vLLM workers.
"""
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
pg = StatelessProcessGroup.create(
host=master_address, port=master_port, rank=rank, world_size=world_size
)
with unpinned_nccl_env():
return PyNcclCommunicator(pg, device=device)
def uid_init_process_group(
nccl_unique_id_bytes: bytes,
rank: int,
world_size: int,
device,
) -> "PyNcclCommunicator":
"""Join the NCCL group from pre-shared ``ncclUniqueId`` bytes.
The torch-free rendezvous alternative to `stateless_init_process_group`: no
TCPStore, and therefore no barrier -- every rank must enter concurrently.
"""
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
with unpinned_nccl_env():
return PyNcclCommunicator.from_unique_id_bytes(
nccl_unique_id_bytes,
rank=rank,
world_size=world_size,
device=device,
)
def _worker_group_rank(rank_offset: int, parallel_config: "ParallelConfig") -> int:
"""Return this worker's rank in a trainer-to-worker process group."""
dp_rank = parallel_config.data_parallel_index
world_size_per_dp = parallel_config.world_size # TP * PP
rank_within_dp = parallel_config.rank
return rank_offset + dp_rank * world_size_per_dp + rank_within_dp
def worker_init_metadata_group(
init_info: NCCLWeightTransferInitInfo,
parallel_config: "ParallelConfig",
) -> "StatelessProcessGroup":
"""Join the trainer-to-worker TCP metadata group without constructing NCCL."""
assert init_info.master_address is not None
assert init_info.master_port is not None
return stateless_init_metadata_group(
init_info.master_address,
init_info.master_port,
_worker_group_rank(init_info.rank_offset, parallel_config),
init_info.world_size,
)
def worker_init_process_group(
init_info: NCCLWeightTransferInitInfo,
parallel_config: "ParallelConfig",
) -> "PyNcclCommunicator":
"""Create the trainer<->worker NCCL group on an inference worker.
Computes a unique rank for this worker across all data-parallel groups and
joins the trainer via whichever rendezvous mode `init_info` carries.
"""
rank = _worker_group_rank(init_info.rank_offset, parallel_config)
device = torch.accelerator.current_device_index()
unique_id_bytes = init_info.nccl_unique_id_bytes
if unique_id_bytes is not None:
return uid_init_process_group(
unique_id_bytes,
rank,
init_info.world_size,
device,
)
# __post_init__ guarantees the TCP pair when there is no unique id.
assert init_info.master_address is not None
assert init_info.master_port is not None
return stateless_init_process_group(
init_info.master_address,
init_info.master_port,
rank,
init_info.world_size,
device=device,
)
def trainer_init(
init_info: NCCLRendezvous | dict,
rank: int = 0,
) -> "PyNcclCommunicator":
"""Initialize NCCL process group for trainer-side weight transfer.
Uses the current CUDA device (torch.accelerator.current_device_index()).
Args:
init_info: Any object carrying the `NCCLRendezvous` fields (a trainer or
worker NCCL init info), or a dict with keys:
- master_address: str
- master_port: int
- world_size: int
rank: This trainer process's rank in the group. The broadcast backends
have a single trainer at rank 0; multi-rank trainers (m2n) occupy
ranks `[0, num_trainer_ranks)` and the workers start after them.
Returns:
PyNcclCommunicator for weight transfer.
"""
if isinstance(init_info, dict):
master_address = init_info["master_address"]
master_port = init_info["master_port"]
world_size = init_info["world_size"]
else:
master_address = init_info.master_address
master_port = init_info.master_port
world_size = init_info.world_size
device = torch.accelerator.current_device_index()
return stateless_init_process_group(
master_address,
master_port,
rank,
world_size,
device,
)
@@ -1,461 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NCCL-based (dense) weight transfer engine."""
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from dataclasses import asdict, dataclass
from typing import TYPE_CHECKING, ClassVar, Self
import torch
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
ParamMeta,
TrainerInitInfo,
TrainerWeightTransferEngine,
VLLMWeightSyncClient,
WeightSource,
WeightTransferEngine,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
worker_init_payload,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_common import (
trainer_init as open_trainer_endpoint,
)
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
DEFAULT_PACKED_NUM_BUFFERS,
packed_nccl_broadcast_consumer,
packed_nccl_broadcast_producer,
)
# NCCLWeightTransferInitInfo is re-exported here for convenience; its canonical
# home is nccl_common, shared with the sparse backend.
__all__ = [
"NCCLWeightTransferInitInfo",
"NCCLTrainerInitInfo",
"NCCLWeightTransferUpdateInfo",
"NCCLWeightTransferEngine",
"NCCLTrainerWeightTransferEngine",
]
@dataclass
class NCCLTrainerInitInfo(TrainerInitInfo):
"""Trainer-side init info for the dense NCCL weight transfer backend.
The sender opens its endpoint as NCCL rank 0, so it needs no `rank_offset`.
`world_size` is the full trainer+worker NCCL group size. `rank` (from
`TrainerInitInfo`) identifies this trainer process; rank 0 is the sender.
The trainer joins over a TCPStore rendezvous (`master_address` +
`master_port`). Torch-free trainers (e.g. JAX) that cannot join a TCPStore
mint an `ncclUniqueId` themselves and drive rank 0 out of band; they ship a
`nccl_unique_id_b64` payload straight to the inference workers'
`init_weight_transfer_engine` (see `NCCLWeightTransferInitInfo`) rather than
going through this engine.
`packed` / buffer sizes are the transfer's wire params. The trainer
propagates them to the worker at `trainer_init` so the two sides cannot
disagree. Note this defaults to packed, unlike the worker-side
`NCCLWeightTransferInitInfo`, whose default only applies when no trainer
ships a value. `backend` is the factory dispatch key."""
backend: ClassVar[str] = "nccl"
master_address: str
master_port: int
world_size: int
packed: bool = True
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
@dataclass
class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Per-round update info for the dense NCCL weight transfer backend.
Whether the transfer is packed (and the buffer geometry) is a must-agree
wire param carried on the init info (`NCCLTrainerInitInfo` /
`NCCLWeightTransferInitInfo`), not here; this carries only the per-round
parameter metadata.
"""
names: list[str]
dtype_names: list[str]
shapes: list[list[int]]
def __post_init__(self):
"""Validate that all lists have the same length."""
num_params = len(self.names)
if len(self.dtype_names) != num_params:
raise ValueError(
f"`dtype_names` should be of the same size as `names`: "
f"got {len(self.dtype_names)} and {len(self.names)}"
)
if len(self.shapes) != num_params:
raise ValueError(
f"`shapes` should be of the same size as `names`: "
f"got {len(self.shapes)} and {len(self.names)}"
)
class NCCLWeightTransferEngine(
WeightTransferEngine[NCCLWeightTransferInitInfo, NCCLWeightTransferUpdateInfo]
):
"""Weight transfer engine using NCCL for communication between trainer and workers.
This implementation uses NCCL broadcast operations to transfer dense
checkpoint-format weights from the trainer (rank 0) to all inference workers
in a process group. Received weights are loaded via the model's
`load_weights` using the layerwise reload lifecycle.
"""
# Define backend-specific dataclass types
init_info_cls = NCCLWeightTransferInitInfo
update_info_cls = NCCLWeightTransferUpdateInfo
def __init__(
self,
config: WeightTransferConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
super().__init__(config, vllm_config, device, model)
self.model_update_group: PyNcclCommunicator | None = None
# Set from the trainer-supplied init info at the handshake; defaults are
# only for the (unreachable) receive-before-init case.
self.packed = False
self.packed_buffer_size_bytes = DEFAULT_PACKED_BUFFER_SIZE_BYTES
self.packed_num_buffers = DEFAULT_PACKED_NUM_BUFFERS
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
"""Initialize NCCL process group with the trainer and record the
trainer-supplied wire params so the worker decodes exactly as the
trainer encodes.
Args:
init_info: NCCL initialization info containing master address, port,
rank offset, world size, and the packed wire params
"""
self.packed = init_info.packed
self.packed_buffer_size_bytes = init_info.packed_buffer_size_bytes
self.packed_num_buffers = init_info.packed_num_buffers
self.model_update_group = worker_init_process_group(
init_info, self.parallel_config
)
def start_weight_update(self) -> None:
"""Initialize layerwise reloading for the incoming checkpoint weights."""
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
initialize_layerwise_reload(self.model)
def finish_weight_update(self) -> None:
"""Finalize layerwise reloading after all weights have been received."""
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
)
finalize_layerwise_reload(self.model, self.model_config)
def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None:
"""Receive weights from trainer via NCCL broadcast.
Whether to use packed broadcasting (and the buffer geometry) is read
from `self.packed` / `self.packed_*`, set at the init handshake from the
trainer's init info, so it is guaranteed to match how the trainer
encoded.
Args:
update_info: NCCL update info containing parameter names, dtypes,
and shapes
"""
if self.model_update_group is None:
raise RuntimeError(
"NCCL weight transfer not initialized. "
"Call init_transfer_engine() first."
)
from vllm.model_executor.model_loader.mtp_validation import (
disable_mtp_completeness_check,
)
with disable_mtp_completeness_check():
if self.packed:
# Build iterator of (name, (shape, dtype)) from update_info
def state_dict_info_iterator():
for name, dtype_name, shape in zip(
update_info.names, update_info.dtype_names, update_info.shapes
):
dtype = getattr(torch, dtype_name)
yield (name, (shape, dtype))
packed_nccl_broadcast_consumer(
iterator=state_dict_info_iterator(),
group=self.model_update_group,
src=0,
post_unpack_func=self.model.load_weights,
buffer_size_bytes=self.packed_buffer_size_bytes,
num_buffers=self.packed_num_buffers,
device=self.device,
)
else:
# Use simple one-by-one broadcasting
for name, dtype_name, shape in zip(
update_info.names, update_info.dtype_names, update_info.shapes
):
dtype = getattr(torch, dtype_name)
weight = torch.empty(shape, dtype=dtype, device=self.device)
self.model_update_group.broadcast(
weight, src=0, stream=torch.cuda.current_stream()
)
self.model.load_weights([(name, weight)])
del weight
def shutdown(self) -> None:
if self.model_update_group is not None:
self.model_update_group.destroy()
self.model_update_group = None
class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerInitInfo]):
"""Trainer-side NCCL weight transfer engine.
On the sender (rank 0) holds the NCCL communicator and drives the full
update round trip: it runs the inference-side `update_weights` concurrently
with the trainer-side broadcast (both rendezvous inside the same NCCL
calls), then finishes the update. Non-sender trainer ranks hold no
communicator; they only iterate the source to stay in the trainer-side
collective (e.g. FSDP `full_tensor()`) and skip the client RPCs and the
broadcast (all guarded on `is_sender`).
`packed` / buffer sizes come from `NCCLTrainerInitInfo`; the sender
propagates them to the worker at `trainer_init` (on the worker-side init
info), so per-round payloads carry only parameter metadata.
"""
init_info_cls = NCCLTrainerInitInfo
def __init__(
self,
*,
client: VLLMWeightSyncClient,
source: WeightSource,
is_sender: bool = True,
packed: bool = True,
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
) -> None:
super().__init__(client=client, source=source, is_sender=is_sender)
self.packed = packed
self.packed_buffer_size_bytes = packed_buffer_size_bytes
self.packed_num_buffers = packed_num_buffers
self.model_update_group: PyNcclCommunicator | None = None
@classmethod
def trainer_init(
cls,
init_info: NCCLTrainerInitInfo,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
) -> Self:
if source is None:
raise ValueError("NCCL trainer weight transfer requires a WeightSource.")
engine = cls(
client=client,
source=source,
is_sender=init_info.is_sender,
packed=init_info.packed,
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
packed_num_buffers=init_info.packed_num_buffers,
)
if not engine.is_sender:
# Non-sender trainer ranks aren't part of the transfer NCCL group and
# don't drive the inference side; they only participate in the
# trainer-side gather during send_weights.
return engine
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
worker_init_info = NCCLWeightTransferInitInfo(
master_address=init_info.master_address,
master_port=init_info.master_port,
rank_offset=1,
world_size=init_info.world_size,
packed=init_info.packed,
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
packed_num_buffers=init_info.packed_num_buffers,
)
# The inference workers block inside init_weight_transfer_engine waiting
# for the NCCL rendezvous, so we kick that off on a side thread while we
# open the trainer endpoint (rank 0); both sides must rendezvous together.
with ThreadPoolExecutor(max_workers=1) as exe:
future = exe.submit(
engine.client.init_weight_transfer_engine,
worker_init_payload(worker_init_info),
)
engine.model_update_group = open_trainer_endpoint(init_info)
future.result() # surface any inference-side init error
return engine
def send_weights(self) -> None:
assert self.source is not None # guaranteed by trainer_init / __init__
source = self.source
# Metadata is declared without gathering. For Megatron it is itself a
# collective, so every rank runs it; only the sender ships it.
meta = source.metadata()
if not self.is_sender:
# Non-sender ranks only join the trainer-side gather collective.
self._broadcast(source, meta)
self._post_send_sync()
return
update_info = NCCLWeightTransferUpdateInfo(
names=[m.name for m in meta],
dtype_names=[str(m.dtype).split(".")[-1] for m in meta],
shapes=[list(m.shape) for m in meta],
)
self.client.start_weight_update()
# update_weights (workers receive) must run concurrently with the
# trainer-side broadcast — both rendezvous inside the same NCCL calls.
exe = ThreadPoolExecutor(max_workers=1)
try:
future = exe.submit(self.client.update_weights, asdict(update_info))
# Cheap best-effort: if update_weights already failed (e.g. a bad
# request rejected before any NCCL call), surface it now instead of
# hanging in broadcast waiting for a peer that will never arrive.
if future.done():
future.result()
self._broadcast(source, meta)
future.result() # surface inference-side errors
finally:
# Never wait for the RPC thread here. If the broadcast raised, the
# worker is still blocked in the matching NCCL call and will never
# return, so joining would turn the error into a permanent hang.
# Let the exception out instead; the transfer group is unusable
# either way and the caller has to tear it down. (The orphaned
# thread is still joined at interpreter exit, so a caller that wants
# to exit cleanly after such a failure must not wait on it.)
exe.shutdown(wait=False)
self.client.finish_weight_update()
self._post_send_sync()
def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None:
"""Iterate the source (materializing each tensor — a collective on all
ranks) and, on the sender, broadcast from rank 0, packed or one-by-one.
Non-sender ranks only replay the iteration to stay in the collective."""
if not self.is_sender:
for _ in source:
pass
return
assert self.model_update_group is not None, (
"trainer_init() must be called before _broadcast()."
)
pairs = self._checked_iter(source, meta)
if self.packed:
packed_nccl_broadcast_producer(
iterator=pairs,
group=self.model_update_group,
src=0,
post_iter_func=lambda item: item[1],
buffer_size_bytes=self.packed_buffer_size_bytes,
num_buffers=self.packed_num_buffers,
)
else:
stream = torch.cuda.current_stream()
for _name, tensor in pairs:
# NCCL sends `numel` elements straight from `data_ptr()`, so a
# non-contiguous view would ship whatever follows its base
# pointer. Keep the copy referenced until the broadcast is
# enqueued. (The packed path linearizes in `pack_tensors`.)
send = tensor if tensor.is_contiguous() else tensor.contiguous()
self.model_update_group.broadcast(send, src=0, stream=stream)
@staticmethod
def _checked_iter(
source: WeightSource, meta: list[ParamMeta]
) -> Iterator[tuple[str, torch.Tensor]]:
"""Yield the source's pairs, checking each against what the worker was
told to expect.
The worker sizes its receive buffers — and in packed mode cuts its chunk
boundaries — from the update info, which is built from `metadata()`. If
iteration disagrees with it, the two sides split the stream differently
and the transfer hangs in NCCL or loads garbage. Checking here costs one
comparison per parameter and turns that into an error naming the first
divergent parameter. Sender-only: under pipeline parallelism a
non-sender's yielded tensor is not meaningful.
"""
sent = 0
for name, tensor in source:
if sent >= len(meta):
raise ValueError(
f"WeightSource yielded more parameters than metadata() "
f"declared ({len(meta)}); first extra is {name!r}."
)
expected = meta[sent]
if (
name != expected.name
or tensor.dtype != expected.dtype
or tuple(tensor.shape) != expected.shape
):
raise ValueError(
"WeightSource metadata() disagrees with iteration at index "
f"{sent}: declared {expected.name!r} "
f"{expected.dtype} {tuple(expected.shape)}, got {name!r} "
f"{tensor.dtype} {tuple(tensor.shape)}. Both channels must "
"enumerate the same parameters in the same order."
)
sent += 1
yield name, tensor
if sent != len(meta):
raise ValueError(
f"WeightSource yielded {sent} parameters but metadata() "
f"declared {len(meta)}; the worker is waiting for the rest."
)
def _post_send_sync(self) -> None:
"""Wait for this rank's transfer work to land before returning.
Broadcasts are only *enqueued* by `send_weights`: the unpacked path on
the current stream, the packed path on the producer's own streams (which
it drains itself). Waiting here lets a caller mutate parameters, or
start the next step on another stream, as soon as `send_weights`
returns, instead of silently depending on same-stream ordering. Every
rank waits: a non-sender's `full_tensor()` gathers feed the sender's
broadcast, so they must have landed before it may touch its shards.
Unlike IPC there is no cross-rank barrier here. Nothing in this backend
outlives the collective it travelled in (IPC's barrier keeps shared
buffers alive until every consumer has opened them), so a barrier would
only add a dependency on the default process group that this backend
otherwise does not have.
"""
if torch.cuda.is_available():
torch.cuda.current_stream().synchronize()
def shutdown(self) -> None:
if self.model_update_group is not None:
self.model_update_group.destroy()
self.model_update_group = None
@@ -1,494 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Packed tensor utilities for efficient weight transfer."""
import math
from collections.abc import Callable, Iterator
from dataclasses import dataclass
from functools import cache
from typing import Any
import torch
from torch.multiprocessing.reductions import reduce_tensor
# Default values for packed tensor transfer.
DEFAULT_PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024 # 1GB
DEFAULT_PACKED_NUM_BUFFERS = 2
# Streams are cached across calls: the caching allocator keys cached blocks by the
# stream they were allocated on, so fresh streams at every call would strand the
# packed buffers' reserved memory per call. See gh-52950.
@cache
def _get_streams(device_idx: int, num_buffers: int) -> tuple[torch.cuda.Stream, ...]:
return tuple(torch.cuda.Stream(device=device_idx) for _ in range(num_buffers))
def unpack_tensor(
packed_tensor: torch.Tensor,
names: list[str],
shapes: list[list[int]],
dtypes: list[torch.dtype],
tensor_sizes: list[int],
) -> list[tuple[str, torch.Tensor]]:
"""Unpack a packed uint8 tensor into a list of named tensors.
The returned tensors are **views** of ``packed_tensor`` (the
``.contiguous()`` call is a no-op on already-contiguous row-slices).
If ``packed_tensor`` lives in storage that may be reused — e.g. a
reused CUDA IPC buffer — callers must clone the results before the
underlying storage is overwritten.
Args:
packed_tensor: The packed torch.uint8 tensor to unpack
names: List of tensor names
shapes: List of tensor shapes
dtypes: List of tensor dtypes
tensor_sizes: List of tensor sizes in bytes
"""
unpacked_tensors = packed_tensor.split(tensor_sizes)
return [
(name, tensor.contiguous().view(dtype).view(shape))
for name, shape, dtype, tensor in zip(names, shapes, dtypes, unpacked_tensors)
]
@dataclass
class PackedChunk:
"""Result of packing tensors into a single contiguous uint8 buffer."""
packed_tensor: torch.Tensor
names: list[str]
shapes: list[list[int]]
dtypes: list[torch.dtype]
tensor_sizes: list[int]
def pack_tensors(
iterator: Iterator[tuple[str, torch.Tensor]],
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
buffer_size_bytes: int,
tensor_list: list[torch.Tensor] | None = None,
current_size: int = 0,
) -> PackedChunk | None:
"""Pack tensors from an iterator into a single contiguous uint8 buffer.
Consumes from the iterator until the accumulated size exceeds
buffer_size_bytes or the iterator is exhausted, then returns a
PackedChunk. Returns None if no tensors were consumed.
Args:
iterator: Iterator of (name, tensor) pairs
post_iter_func: Applied to each item before linearizing to uint8
buffer_size_bytes: Max bytes before flushing
tensor_list: Pre-existing tensor list to append to (for NCCL
multi-buffer reuse). If None, a fresh list is created.
current_size: Byte count already accumulated in tensor_list
"""
if tensor_list is None:
tensor_list = []
names: list[str] = []
shapes: list[list[int]] = []
dtypes: list[torch.dtype] = []
tensor_sizes: list[int] = []
total_bytes = current_size
while True:
try:
item = next(iterator)
except StopIteration:
break
name, orig_tensor = item
# Apply post processing and convert to linearized uint8 tensor
tensor = post_iter_func(item).contiguous().view(-1).view(torch.uint8)
if tensor.numel() > buffer_size_bytes:
import warnings
warnings.warn(
f"Tensor '{name}' has size {tensor.numel()} bytes, which "
f"exceeds buffer_size_bytes={buffer_size_bytes}.",
stacklevel=2,
)
tensor_list.append(tensor)
names.append(name)
shapes.append(list(orig_tensor.shape))
dtypes.append(orig_tensor.dtype)
tensor_sizes.append(tensor.numel())
total_bytes += tensor.numel()
if total_bytes > buffer_size_bytes:
break
if not tensor_list:
return None
packed = torch.cat(tensor_list, dim=0)
del tensor_list
return PackedChunk(
packed_tensor=packed,
names=names,
shapes=shapes,
dtypes=dtypes,
tensor_sizes=tensor_sizes,
)
# ── NCCL packed broadcast ──────────────────────────────────────────────
def packed_nccl_broadcast_producer(
iterator: Iterator[tuple[str, torch.Tensor]],
group: Any,
src: int,
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
) -> None:
"""Broadcast tensors in a packed manner from trainer to workers.
Args:
iterator: Iterator of model parameters. Returns a tuple of (name, tensor)
group: Process group (PyNcclCommunicator)
src: Source rank (0 in current implementation)
post_iter_func: Function to apply to each (name, tensor) pair before
packing, should return a tensor
buffer_size_bytes: Size in bytes for each packed tensor buffer.
Both producer and consumer must use the same value.
num_buffers: Number of buffers for double/triple buffering.
Both producer and consumer must use the same value.
"""
streams = _get_streams(torch.accelerator.current_device_index(), num_buffers)
# Keep references to in-flight chunks so their packed_tensors
# aren't freed while an async broadcast is still reading them.
in_flight: list[PackedChunk | None] = [None] * num_buffers
buffer_idx = 0
while True:
# Synchronize the current stream
streams[buffer_idx].synchronize()
# Previous chunk on this buffer slot is now safe to free
in_flight[buffer_idx] = None
# Start tasks for the new buffer in a new stream
with torch.cuda.stream(streams[buffer_idx]):
chunk = pack_tensors(iterator, post_iter_func, buffer_size_bytes)
if chunk is None:
break
# Pack the tensors and call broadcast collective
group.broadcast(chunk.packed_tensor, src=src)
# Hold reference until this stream is synchronized
in_flight[buffer_idx] = chunk
# Move to the next buffer
buffer_idx = (buffer_idx + 1) % num_buffers
# Drain every slot before returning.
for stream in streams:
stream.synchronize()
def packed_nccl_broadcast_consumer(
iterator: Iterator[tuple[str, tuple[list[int], torch.dtype]]],
group: Any,
src: int,
post_unpack_func: Callable[[list[tuple[str, torch.Tensor]]], None],
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
device: torch.device | str = "cuda",
) -> None:
"""Consume packed tensors and unpack them into a list of tensors.
Args:
iterator: Iterator of parameter metadata. Returns (name, (shape, dtype))
group: Process group (PyNcclCommunicator)
src: Source rank (0 in current implementation)
post_unpack_func: Function to apply to each list of (name, tensor) after
unpacking
buffer_size_bytes: Size in bytes for each packed tensor buffer.
Both producer and consumer must use the same value.
num_buffers: Number of buffers for double/triple buffering.
Both producer and consumer must use the same value.
device: Device for the receive buffers. Must match the device the NCCL
communicator was created on (the worker's assigned device).
"""
target_packed_tensor_size = buffer_size_bytes
streams = _get_streams(torch.accelerator.current_device_index(), num_buffers)
buffer_idx = 0
packing_tensor_meta_data: list[list[tuple[str, list[int], torch.dtype, int]]] = [
[] for _ in range(num_buffers)
]
packing_tensor_sizes: list[int] = [0 for _ in range(num_buffers)]
packed_tensors: list[torch.Tensor] = [
torch.empty(0, dtype=torch.uint8, device=device) for _ in range(num_buffers)
]
while True:
# Synchronize the current stream
streams[buffer_idx].synchronize()
with torch.cuda.stream(streams[buffer_idx]):
# Initialize the packing tensor meta data
packing_tensor_meta_data[buffer_idx] = []
packing_tensor_sizes[buffer_idx] = 0
try:
# Form a packed tensor
while True:
name, (shape, dtype) = next(iterator)
tensor_size = math.prod(shape) * dtype.itemsize
packing_tensor_meta_data[buffer_idx].append(
(name, shape, dtype, tensor_size)
)
packing_tensor_sizes[buffer_idx] += tensor_size
if packing_tensor_sizes[buffer_idx] > target_packed_tensor_size:
break
# Create a packed tensor and broadcast it
packed_tensors[buffer_idx] = torch.empty(
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device=device
)
group.broadcast(packed_tensors[buffer_idx], src=src)
# Load the packed tensor into the model
names, shapes, dtypes, tensor_sizes = zip(
*packing_tensor_meta_data[buffer_idx]
)
post_unpack_func(
unpack_tensor(
packed_tensors[buffer_idx],
list(names),
list(shapes),
list(dtypes),
list(tensor_sizes),
)
)
# Move to the next buffer
buffer_idx = (buffer_idx + 1) % num_buffers
except StopIteration:
# Do the last broadcast if there are remaining tensors
if len(packing_tensor_meta_data[buffer_idx]) > 0:
# Create a packed tensor and broadcast it
packed_tensors[buffer_idx] = torch.empty(
packing_tensor_sizes[buffer_idx],
dtype=torch.uint8,
device=device,
)
group.broadcast(packed_tensors[buffer_idx], src=src)
# Load the packed tensor into the model
names, shapes, dtypes, tensor_sizes = zip(
*packing_tensor_meta_data[buffer_idx]
)
post_unpack_func(
unpack_tensor(
packed_tensors[buffer_idx],
list(names),
list(shapes),
list(dtypes),
list(tensor_sizes),
)
)
break
# Drain every slot before returning.
for stream in streams:
stream.synchronize()
# ── IPC packed transfer ────────────────────────────────────────────────
@dataclass
class PackedIpcChunk:
"""Metadata and IPC handle for a single packed chunk."""
names: list[str]
shapes: list[list[int]]
dtype_names: list[str]
tensor_sizes: list[int]
ipc_handle: dict[str, tuple]
def packed_ipc_producer(
iterator: Iterator[tuple[str, torch.Tensor]],
gpu_uuid: str,
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
) -> Iterator[PackedIpcChunk]:
"""Pack tensors into a reusable IPC buffer and yield handles.
Allocates a single GPU buffer of ``buffer_size_bytes`` and registers
it for IPC once via ``reduce_tensor``. Each chunk's packed data is
copied into this buffer before yielding, so only one IPC-shared
allocation exists for the lifetime of the transfer.
Callers **must** ensure the consumer has finished reading the buffer
(e.g. ``ray.get`` returned) before resuming the generator for the
next chunk.
Args:
iterator: Iterator of (name, tensor) pairs.
gpu_uuid: Physical GPU UUID string for this rank.
post_iter_func: Applied to each (name, tensor) before packing.
buffer_size_bytes: Exact capacity of the reusable IPC buffer.
Every chunk is guaranteed to fit within this size. A
``ValueError`` is raised if any single tensor exceeds it.
"""
ipc_buffer = torch.empty(buffer_size_bytes, dtype=torch.uint8, device="cuda")
_, ipc_args = reduce_tensor(ipc_buffer)
names: list[str] = []
shapes: list[list[int]] = []
dtypes: list[torch.dtype] = []
tensor_sizes: list[int] = []
total_bytes = 0
for name, orig_tensor in iterator:
flat = (
post_iter_func((name, orig_tensor)).contiguous().view(-1).view(torch.uint8)
)
if flat.numel() > buffer_size_bytes:
raise ValueError(
f"Tensor '{name}' has size {flat.numel()} bytes, "
f"which exceeds buffer_size_bytes={buffer_size_bytes}. "
f"Increase buffer_size_bytes to at least {flat.numel()}."
)
if total_bytes and total_bytes + flat.numel() > buffer_size_bytes:
# Drain queued copies so the consumer sees a fully-written buffer.
torch.cuda.current_stream().synchronize()
yield PackedIpcChunk(
names=names,
shapes=shapes,
dtype_names=[str(d).split(".")[-1] for d in dtypes],
tensor_sizes=tensor_sizes,
ipc_handle={gpu_uuid: ipc_args},
)
# Rebind to fresh lists so the yielded chunk's metadata is
# not mutated while the consumer is still reading.
names, shapes, dtypes, tensor_sizes = [], [], [], []
total_bytes = 0
ipc_buffer[total_bytes : total_bytes + flat.numel()].copy_(flat)
names.append(name)
shapes.append(list(orig_tensor.shape))
dtypes.append(orig_tensor.dtype)
tensor_sizes.append(flat.numel())
total_bytes += flat.numel()
if total_bytes:
torch.cuda.current_stream().synchronize()
yield PackedIpcChunk(
names=names,
shapes=shapes,
dtype_names=[str(d).split(".")[-1] for d in dtypes],
tensor_sizes=tensor_sizes,
ipc_handle={gpu_uuid: ipc_args},
)
class PackedBufferImporter:
"""One-slot cache for the consumer-side mapping of a packed IPC buffer.
torch's cross-process refcount pairs one ``reduce_tensor`` export with
one consumer rebuild-release cycle. The producer exports its buffer once
and ships the same args with every chunk, so rebuilding and releasing
per chunk decrements the counter once per chunk, drives it negative,
and the dropped buffer is then never reclaimable by ``ipc_collect`` on
the producer side (it leaks one buffer per transfer). Caching the
rebuilt buffer keyed by the rebuild args restores the pairing: chunks
of one transfer reuse the mapping (it aliases producer memory, so it
always reads the current chunk's bytes), and replacing the entry when
the next export arrives — or ``close()`` — releases the previous
mapping exactly once.
The importer that consumes a multi-chunk transfer must be one object
reused across chunks; a fresh importer per chunk reintroduces the
per-chunk release. ``IPCWeightTransferEngine`` owns one instance per
engine and closes it on shutdown.
"""
def __init__(self) -> None:
self._entry: tuple[tuple, torch.Tensor] | None = None
def rebuild(self, list_args: list) -> torch.Tensor:
from torch.multiprocessing.reductions import rebuild_cuda_tensor
key = tuple(list_args)
if self._entry is not None and self._entry[0] == key:
return self._entry[1]
packed = rebuild_cuda_tensor(*list_args)
self._entry = (key, packed)
return packed
def close(self) -> None:
"""Release the held mapping (one producer-side refcount decrement)."""
self._entry = None
def packed_ipc_consumer(
ipc_handle: dict[str, tuple],
names: list[str],
shapes: list[list[int]],
dtype_names: list[str],
tensor_sizes: list[int],
device_index: int,
importer: "PackedBufferImporter | None" = None,
) -> list[tuple[str, torch.Tensor]]:
"""Unpack a single packed IPC chunk into named tensors.
Reconstructs the packed buffer via rebuild_cuda_tensor, unpacks
into individual tensors, and clones each into independent storage
before returning.
The clone is intentional: the producer reuses one IPC buffer across
chunks, so any tensor view that aliases the buffer would observe the
*next* chunk's bytes as soon as the producer's generator is resumed.
Callers that retain references past their own update_weights call
(notably vLLM's layerwise reload, which buffers ``bound_args`` for
replay in ``_layerwise_process``) would otherwise replay against
stale data and silently corrupt multi-chunk weight transfers.
Args:
ipc_handle: Mapping of GPU UUID to rebuild_cuda_tensor args tuple
names: Parameter names in the packed buffer
shapes: Parameter shapes
dtype_names: Parameter dtype name strings (e.g. "float16")
tensor_sizes: Size in bytes of each parameter in the packed buffer
device_index: Local CUDA device index
importer: Import cache that must be shared across all chunks of one
export so the buffer mapping is opened and released exactly once
(see ``PackedBufferImporter``). ``None`` builds an ephemeral one,
which is only safe for single-chunk consumption.
"""
props = torch.cuda.get_device_properties(device_index)
physical_gpu_id = str(props.uuid)
if physical_gpu_id not in ipc_handle:
raise ValueError(
f"IPC handle not found for GPU UUID {physical_gpu_id}. "
f"Available UUIDs: {list(ipc_handle.keys())}"
)
args = ipc_handle[physical_gpu_id]
list_args = list(args)
list_args[6] = device_index
if importer is None:
importer = PackedBufferImporter()
packed = importer.rebuild(list_args)
content_size = sum(tensor_sizes)
packed = packed[:content_size]
dtypes = [getattr(torch, dn) for dn in dtype_names]
return [
(name, t.clone())
for name, t in unpack_tensor(packed, names, shapes, dtypes, tensor_sizes)
]
@@ -1,352 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared pieces of the sharded-RDT backend: the op-chain allowlist, buffer
sizing and the minimum Ray version, plus ``RdtRouter`` — consumer-only, since
routing is a consumer decision.
The gather-group partition itself is ``base.layerwise_groups``: it defines what a
group index means for any ``WeightSource``, not just this transport.
"""
from collections.abc import Callable
from typing import Any
import torch
# The op-chain contract, with two enforcers. The consumer's ``FakeRDTTensor``
# intercepts exactly these methods during the bake and records the string name;
# anything else reaches ``__torch_dispatch__`` and raises, so a loader needing
# real data fails at init rather than transferring wrong bytes. The producer
# replays a chain as ``getattr(tensor, op)(...)`` and refuses anything outside
# ``ALLOWED_OPS``, so a spoofed consumer cannot invoke arbitrary methods.
#
# Every entry must be a pure view; ``to`` in particular must never be added, as
# it is the dtype/device escape the bake exists to reject. Both sets derive from
# this one table so they cannot disagree.
SUPPORTED_OPS: dict[Callable, str] = {
torch.Tensor.narrow: "narrow",
torch.Tensor.view: "view",
torch.Tensor.reshape: "reshape",
torch.Tensor.__getitem__: "__getitem__",
torch.Tensor.unsqueeze: "unsqueeze",
torch.Tensor.squeeze: "squeeze",
torch.Tensor.transpose: "transpose",
torch.Tensor.t: "t",
torch.Tensor.permute: "permute",
torch.Tensor.flatten: "flatten",
torch.Tensor.contiguous: "contiguous",
torch.Tensor.chunk: "chunk",
# Multi-return like chunk: one child per output, trailing __getitem__(i).
torch.Tensor.unbind: "unbind",
}
ALLOWED_OPS = frozenset(SUPPORTED_OPS.values())
# Ray Direct Transport landed in halves: the actor opt-in
# ``enable_tensor_transport`` in 2.49, and ``ray.experimental``'s
# ``register_nixl_memory`` / ``set_target_for_ref`` — which this transport calls
# directly — in 2.55. The floor is the tested version rather than the earliest
# one that could import; 2.55 has never been run against.
RDT_MIN_RAY_VERSION = "2.56.0"
def check_ray_rdt_version() -> None:
"""Refuse an installed Ray older than the one this backend is tested on.
vLLM does not depend on Ray, so there is no pin to carry this. Without the
check the failure is an opaque option-validation error out of
``.options(enable_tensor_transport=True)`` (below 2.49) or an ImportError
raised deep in the first pull, long after init reported success (below 2.55).
Raises:
ValueError: the installed Ray predates ``RDT_MIN_RAY_VERSION``.
"""
import importlib.metadata
from packaging import version
required = version.parse(RDT_MIN_RAY_VERSION)
current = version.parse(importlib.metadata.version("ray"))
if current < required:
raise ValueError(
f"The 'sharded_rdt' weight transfer backend requires Ray "
f">= {required}, the version it is tested against; Ray Direct "
f"Transport (ray.experimental.register_nixl_memory / "
f"set_target_for_ref) needs at least 2.55. Found {current}. "
f"Run `pip install -U 'ray>={required}'`."
)
def register_nixl_memory(tensor: torch.Tensor) -> None:
"""Register a tensor with Ray using the platform's NIXL package."""
from vllm.distributed.nixl_utils import alias_nixl_for_ray
alias_nixl_for_ray()
from ray.experimental import register_nixl_memory as ray_register_nixl_memory
ray_register_nixl_memory(tensor)
def assign_producer_indices(
num_producers: int, num_consumers: int, consumer_idx: int
) -> list[int]:
"""Producers (global indices) that consumer ``consumer_idx`` binds."""
p = max(1, num_producers)
c = max(1, num_consumers)
if p >= c:
return list(range(consumer_idx * p // c, (consumer_idx + 1) * p // c))
return [consumer_idx * p // c]
class RdtRouter:
"""Decides which producer serves each weight name, and — once bound — sends.
Ownership is per NAME. ``owner_sets`` holds the few distinct producer sets
that occur, and ``name_owner_class[i]`` indexes it for ``names[i]``; a name
determines both its owner set and its gather group, so neither appears in the
routing API. Empty tables mean every producer holds everything.
Any placement is expressible this way: a pipeline stage is a set of names
sharing one owner set, an expert is a name whose owner set is a single rank,
and a group produced by two stages is just two classes inside one group.
Both engines derive the same tables from the same wire data, so they agree on
who serves what. Disagreement is not a wrong answer but a hang or a loud
misroute: a pull sent to a producer that never gathered the name trips its
served-names guard.
Routing is consumer-only. The trainer publishes what it holds and answers
whatever arrives; it never asks who serves what.
"""
def __init__(
self,
num_producers: int,
num_consumers: int,
owner_sets: list[list[int]] | None = None,
name_owner_class: list[int] | None = None,
names: list[str] | None = None,
group_lens: list[int] | None = None,
workers_per_replica: int = 0,
) -> None:
"""Build the routing tables from the wire data both engines receive.
The four table arguments travel together and default together: with all
of them empty the router degrades to "every producer holds everything".
Args:
num_producers: Trainer ranks. Owner indices are positions in this
range, and ``validate`` rejects any that fall outside it.
num_consumers: Inference workers across the whole fleet. Fixes
the id space; the block carve uses ``workers_per_replica``
of it.
owner_sets: The distinct producer sets that occur, one row per
owner class; each row is deduplicated and sorted here. Empty
means a single class owning every producer.
name_owner_class: Parallel to ``names`` — ``name_owner_class[i]``
indexes ``owner_sets`` for ``names[i]``. A name with no entry
falls back to class 0.
names: Every parameter name, in group-major order: concatenating
the gather groups reproduces this list exactly. The other two
tables are keyed by this order.
group_lens: Length of each gather group, consecutive over
``names``, so they sum to ``len(names)``. Fixes name -> group
and the per-group owner union the free barrier fans out to.
workers_per_replica: Consumers per inference DEPLOYMENT. The block
carve spreads this many consumers over an owner set and every
deployment reuses that carve, so the same worker of each
deployment resolves one producer (see ``producer_for``). 0
means one deployment: carve over the whole fleet.
Raises:
ValueError: ``workers_per_replica`` does not divide
``num_consumers``, so the deployments are not uniform and two
workers of one deployment would share a block index.
"""
self.num_producers = max(1, num_producers)
self.num_consumers = max(1, num_consumers)
# Width of the block carve: one deployment's worth of consumers, so
# deployments overlay rather than spread. See ``producer_for``.
self._block_consumers = min(
self.num_consumers, max(1, workers_per_replica or self.num_consumers)
)
if self.num_consumers % self._block_consumers:
raise ValueError(
f"workers_per_replica={workers_per_replica} does not divide "
f"num_consumers={self.num_consumers}; the inference deployments "
f"must be uniform for the per-deployment block carve to be "
f"well defined."
)
self._owner_sets = (
[sorted(set(row)) for row in owner_sets]
if owner_sets
else [list(range(self.num_producers))]
)
names = list(names or [])
classes = list(name_owner_class or [])
self._class_of = {
n: (classes[i] if i < len(classes) else 0) for i, n in enumerate(names)
}
# name -> gather group, and the per-group owner union the free barrier
# fans out to. Both are pure functions of the tables, so precompute once.
self._group_of: dict[str, int] = {}
self._group_owners: list[list[int]] = []
pos = 0
for gi, glen in enumerate(group_lens or []):
union: set[int] = set()
for n in names[pos : pos + glen]:
self._group_of[n] = gi
c = self._class_of.get(n, 0)
# Bounds-checked: a bad class must surface from validate() as a
# named ValueError, not as an IndexError from this precompute.
if 0 <= c < len(self._owner_sets):
union.update(self._owner_sets[c])
pos += glen
self._group_owners.append(sorted(union))
# Bound at init on the consumer; the rule methods never touch these.
self._actors: list[Any] = []
self._produce_methods: list[Any] = []
self.consumer_id = 0
# ---------------- rules (pure; safe to use unbound) ----------------
def class_of(self, name: str) -> int:
"""The name's owner class — the planner's bucketing key, since all names
of a chunk must share one producer."""
return self._class_of.get(name, 0)
def owners(self, name: str) -> list[int]:
"""Every producer holding ``name``."""
return list(self._owner_sets[self.class_of(name)])
def group_owners(self, group_idx: int) -> list[int]:
"""Every producer holding any name of ``group_idx``.
The one group-keyed rule, because the free barrier is per group: a
consumer signals ``free_group(gi)`` at each of these and the producer
counts signals against its live-consumer total. Names route; groups free.
"""
if not self._group_owners:
return list(range(self.num_producers))
return list(self._group_owners[group_idx])
def producer_for(self, consumer_id: int, name: str) -> int:
"""The single producer ``consumer_id`` pulls ``name`` from.
Blocks consumers across the name's owner set with the same rule that
binds producers globally, then rotates by the name's GROUP index so a
consumer spreads its groups over its block instead of hammering one NIC.
Rotating per group (not per name) keeps every name of a chunk on one
producer, which is what lets a chunk be a single pull.
The carve is over ONE DEPLOYMENT (this consumer's index within its own
deployment, over ``workers_per_replica`` of them), so the same worker of
every deployment resolves the same producer for every name. Carving over
the whole fleet instead spreads the multi-owner names of different
deployments onto different producers while single-owner names still
route by ownership, so a producer serves several distinct worker
indices, each needing its own serve ring, and none of them can share a
slot. With one deployment the width is the fleet, so this is the plain
block rule.
"""
own = self.owners(name)
if not own:
raise ValueError(f"{name!r} has no owner")
block = assign_producer_indices(
len(own), self._block_consumers, consumer_id % self._block_consumers
)
return own[block[self._group_of.get(name, 0) % len(block)]]
def validate(self) -> None:
"""Check the ownership tables can be served.
Raises:
ValueError: an owner set is empty or out of range, or a class index
does not resolve.
"""
for c, row in enumerate(self._owner_sets):
if not row:
raise ValueError(f"owner set {c} is empty: no producer holds it")
bad = [p for p in row if not 0 <= p < self.num_producers]
if bad:
raise ValueError(f"owner set {c} out of range: {bad}")
for name, c in self._class_of.items():
if not 0 <= c < len(self._owner_sets):
raise ValueError(
f"{name!r} has owner class {c}, but only "
f"{len(self._owner_sets)} owner set(s) were shipped"
)
# ---------------- send surface (needs the binding) ----------------
def bind(self, actors: list, produce_methods: list, consumer_id: int) -> None:
"""Attach this consumer's producer handles. Rebuilt wholesale per init,
never appended to: every owner index is a position in these lists, so a
rejoining engine that re-inits must not shift them."""
if len(actors) != len(produce_methods):
raise ValueError("actors and produce_methods must be parallel")
self._actors = list(actors)
self._produce_methods = list(produce_methods)
self.consumer_id = consumer_id
def _bound(self) -> None:
if not self._produce_methods:
raise RuntimeError("RdtRouter has no producers bound; call bind() first.")
def pull(self, owner: int, keys: list, seq: int) -> Any:
"""Issue one packed pull to ``owner``.
The consumer id keys the producer's serve ring; without it every worker
is served out of ring 0 and concurrent pulls overwrite each other's blob.
``seq`` is this call's index in the stream to ``owner`` (see ``_Chunk``):
it selects the serve slot in ISSUE order, so a slot is never repacked
while the read of its previous contents is still in flight.
"""
self._bound()
return self._produce_methods[owner].remote(
keys, consumer_id=self.consumer_id, seq=seq
)
def free_group(self, group_idx: int) -> list:
"""Signal the group done at every producer holding any of its names."""
self._bound()
return [
self._actors[p].free_group.remote(group_idx)
for p in self.group_owners(group_idx)
]
def reserve_serve_buffers(
self, bytes_by_producer: list[int], plan_digests: list[str] | None = None
) -> list:
"""Ask each producer to pre-register a serve ring sized to the most this
consumer will pull from it.
``plan_digests[p]`` describes the chunks this consumer pulls from
producer ``p``, in pull order. A producer that shares one serve ring
across the consumers of several deployments compares it across them, so
a fleet whose deployments are not identical fails at init instead of
stalling mid-sync. A producer that shares nothing ignores it.
"""
self._bound()
digests: list = (
list(plan_digests) if plan_digests else [None] * len(bytes_by_producer)
)
return [
self._actors[p].reserve_serve_buffer.remote(
self.consumer_id, nb, digests[p]
)
for p, nb in enumerate(bytes_by_producer)
if nb > 0
]
def buffer_alloc_bytes(nbytes: int, presize: int = 0) -> int:
"""Size a NIXL buffer / ring slot for ``nbytes``: the max of the request, an
optional ``presize`` floor, and a coarse 256MB round-up, so the buffer is
allocated ONCE and never regrows. Regrowth is a correctness hazard, not just a
perf one -- see ``buffer_presize_gb``. Shared by the consumer's receive buffers
and the producer's serve rings."""
return max(nbytes, presize, -(-nbytes // (256 << 20)) * (256 << 20))
File diff suppressed because it is too large Load Diff
@@ -1,300 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Op-chain recording for the sharded-RDT backend.
The consumer asks the trainer for the exact slice a worker consumes, described as
an op chain replayed on the trainer's live tensor. ``FakeRDTTensor`` builds the
chain by intercepting the model's own weight loaders; ``copy_`` is its data sink,
``BakeSink`` — the dry run records how each slice would be fetched and where it
lands.
Chains are built against meta tensors, so nothing here touches Ray or a GPU.
"""
from collections import defaultdict
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any
import torch
from vllm.distributed.weight_transfer.sharded_rdt_common import SUPPORTED_OPS
# ("op_name", positional_args, sorted_kwargs_items). Entries must be hashable so
# the chain can serve as a FetchKey.
OpSpec = tuple[str, tuple[Any, ...], tuple[tuple[str, Any], ...]]
OpChain = tuple[OpSpec, ...]
FetchKey = tuple[str, OpChain]
def _freeze_kwargs(kwargs: dict[str, Any]) -> tuple[tuple[str, Any], ...]:
"""Sort kwargs into a tuple of items for hashable storage in OpSpec."""
return tuple(sorted(kwargs.items()))
class _UnsupportedFakeOp(NotImplementedError):
"""Raised when a weight loader does something to a FakeRDTTensor that cannot
be expressed as a slice request.
Surfaced as NotImplementedError so callers can distinguish "this backend
can't handle this loader" from genuine bugs.
"""
@dataclass
class _Scatter:
"""One recorded scatter: pull ``src``, copy it into ``layer``'s
``param_name`` at the recorded strided region. The bake's output unit; see
the engine module's Data flow.
Self-contained so replay needs no lookups. Destination geometry is read off
the meta view and rebuilt each sync as ``as_strided(shape, stride, offset)``
-- ``layer`` is resolved to a param at replay time, never baked, since every
sync re-materializes fresh tensors.
``shape`` and ``dtype`` also size the slice ON THE WIRE, so ``dtype`` is the
PRODUCED dtype (the fake's after its op chain), not the source name's:
``view(dtype)`` is allowlisted, and taking the source's would size the slice
with the wrong itemsize and carve the packed blob differently on the two
sides.
"""
layer: Any
param_name: str
src: FetchKey
offset: int
shape: tuple[int, ...]
stride: tuple[int, ...]
dtype: torch.dtype
def _meta_copy_(dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor:
"""Fire ``dest.copy_`` from a zero-storage meta source of ``src``'s geometry.
Moves no data but still counts against the layer's loaded numel, which drives
``_layerwise_process``; skipping it would leave the layer looking unloaded
forever.
A non-meta ``dest`` is one the layerwise reload never moved there (the
``SKIP_LOAD_TENSORS`` set, e.g. GLM's router bias). Torch forbids
``real.copy_(meta)``, and skipping the count is consistent -- ``get_layer_size``
excludes the same set. The caller still RECORDS the copy, so the replay writes
the real param.
"""
if not dest.is_meta:
return dest
meta_src = torch.empty(src.shape, dtype=src.dtype, device="meta")
with torch._C.DisableTorchFunctionSubclass():
return dest.copy_(meta_src)
@dataclass
class BakeSink:
"""``copy_`` sink for the dry-run bake: record how each slice would be fetched
and where it would land, and move nothing.
``_install_recording_stamps`` sets ``current = (leaf_module, param_name)``
around each loader so ``accept_copy`` can attribute the copy. An unstamped
``copy_`` cannot be attributed and stays unrecorded, so its module fails the
coverage gate and takes the plain load. ``copies_by_layer`` is keyed by module
object, so iterating it yields each leaf module once.
"""
copies_by_layer: "dict[Any, list[_Scatter | None]]" = field(
default_factory=lambda: defaultdict(list)
)
current: "tuple[Any, str] | None" = None
# Source names whose ``copy_`` fired during the bake. Names not here never
# moved data (e.g. experts owned by another EP rank) and are skipped; names
# here but unbaked fail the plan build.
copied_names: "set[str]" = field(default_factory=set)
def accept_copy(self, dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor:
self.copied_names.add(src._name)
if self.current is not None:
layer, param_name = self.current
if dest.numel() != src.numel():
# The packed layout sizes each slice from the DESTINATION shape,
# while the producer packs what the replayed chain produces. A
# broadcasting copy_ makes those two lengths differ, and neither
# side can notice: the consumer's buffer view is exactly
# prod(dest.shape) elements, so it reshapes cleanly over bytes
# the producer laid out at different offsets. Refuse at bake.
raise _UnsupportedFakeOp(
f"FakeRDTTensor: broadcasting copy_ into {param_name!r} of "
f"{type(layer).__name__} ({src.numel()} elements from "
f"{src._name!r} into {dest.numel()}). The sharded RDT "
"backend sends exactly the slice the loader asks for, so "
"the source and destination must have equal numel."
)
self.copies_by_layer[layer].append(
_Scatter(
layer=layer,
param_name=param_name,
src=src._key(),
offset=dest.storage_offset(),
shape=tuple(dest.shape),
stride=tuple(dest.stride()),
dtype=src.dtype,
)
)
return _meta_copy_(dest, src)
class FakeRDTTensor(torch.Tensor):
"""Zero-storage tensor that records how to fetch a weight slice.
``_make_wrapper_subclass`` gives it shape/dtype/device without storage. Every
op in ``SUPPORTED_OPS`` returns a child with the spec appended; ``copy_``
delegates to the installed sink. Anything else reaches ``__torch_dispatch__``
and raises ``_UnsupportedFakeOp``, so failures are loud rather than silently
fetching the wrong bytes.
"""
# Declared at class scope so mypy can infer attribute types on
# instances built via ``_make_wrapper_subclass`` (where ``__new__``
# returns a tensor it can't annotate as ``self``).
_name: str
_ops: OpChain
# Handles this fake's copy_. ``None`` only on bare construction.
_sink: "BakeSink | None"
@staticmethod
def __new__(
cls,
name: str,
shape: torch.Size,
dtype: torch.dtype,
device: torch.device,
ops: OpChain = (),
sink: "BakeSink | None" = None,
) -> "FakeRDTTensor":
t = torch.Tensor._make_wrapper_subclass(
cls,
shape,
dtype=dtype,
device=device,
requires_grad=False,
)
t._name = name
t._ops = tuple(ops)
t._sink = sink
return t
def _key(self) -> FetchKey:
return (self._name, self._ops)
def _make_child(
self,
new_shape: torch.Size,
new_dtype: torch.dtype,
*new_ops: OpSpec,
) -> "FakeRDTTensor":
"""Append one or more ops to the chain and return a fresh child.
Variadic so multi-return ops (e.g. chunk) can append both the base
op and an indexing op in a single call.
"""
return FakeRDTTensor(
name=self._name,
shape=new_shape,
dtype=new_dtype,
device=self.device,
ops=self._ops + new_ops,
sink=self._sink,
)
def _meta(self) -> torch.Tensor:
"""A zero-storage meta tensor of this fake's shape/dtype, so PyTorch
itself computes post-op geometry rather than us reimplementing it."""
return torch.empty(self.shape, dtype=self.dtype, device="meta")
@classmethod
def __torch_function__(
cls,
func,
types,
args=(),
kwargs=None,
):
kwargs = kwargs or {}
# Checked before the allowlist: copy_ must not be recorded into the chain.
if func is torch.Tensor.copy_:
dest = args[0]
src = args[1] if len(args) > 1 else kwargs.get("src")
if isinstance(src, cls) and src._sink is not None:
return src._sink.accept_copy(dest, src)
# Allowlisted slice/view/shape ops: append to chain and return child.
op_name = SUPPORTED_OPS.get(func)
if op_name is not None:
self_ = args[0]
if isinstance(self_, cls):
rest = tuple(args[1:])
return cls._intercept(self_, func, op_name, rest, kwargs)
# Metadata reads are answered by the wrapper subclass and stop here; ops
# that need data fall through to __torch_dispatch__, which raises.
with torch._C.DisableTorchFunctionSubclass():
return func(*args, **kwargs)
@classmethod
def _intercept(
cls,
self_: "FakeRDTTensor",
func: Callable,
op_name: str,
args: tuple,
kwargs: dict,
):
"""Append the op and return a child, or a tuple of children for
multi-return ops. Each child's geometry comes from running the op on a
meta tensor."""
meta = self_._meta()
with torch._C.DisableTorchFunctionSubclass():
meta_result = func(meta, *args, **kwargs)
base_op: OpSpec = (op_name, tuple(args), _freeze_kwargs(kwargs))
# Single-tensor result: one child carrying base_op alone.
if isinstance(meta_result, torch.Tensor):
return self_._make_child(meta_result.shape, meta_result.dtype, base_op)
# Multi-return: one child per output, each carrying base_op plus a
# trailing __getitem__(i) so the replay can index the result.
if isinstance(meta_result, tuple | list):
return tuple(
self_._make_child(
m.shape,
m.dtype,
base_op,
("__getitem__", (i,), ()),
)
for i, m in enumerate(meta_result)
)
# Non-tensor result (e.g. .item() snuck into the allowlist).
raise _UnsupportedFakeOp(
f"FakeRDTTensor: {op_name!r} returned a non-tensor "
f"({type(meta_result).__name__}); cannot defer."
)
@classmethod
def __torch_dispatch__(cls, func, types, args=(), kwargs=None):
kwargs = kwargs or {}
# A fake at the aten level means an unsupported loader op. Name the op and
# the chain; materializing instead would mask a correctness bug.
for arg in (*args, *kwargs.values()):
if isinstance(arg, cls):
raise _UnsupportedFakeOp(
f"FakeRDTTensor: unsupported op {func} reached "
f"__torch_dispatch__ on fake {arg._name!r} "
f"(chain={arg._ops}). Supported ops are: "
f"{sorted(SUPPORTED_OPS.values())}, plus copy_. "
"Loaders that need .to(), .float(), .item(), arithmetic, "
"bool-mask indexing, or .data access are not supported by "
"the sharded RDT backend."
)
return func(*args, **kwargs)
File diff suppressed because it is too large Load Diff
@@ -1,377 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sparse NCCL weight transfer engine.
Sparse patches use checkpoint names, shapes, and flat indices. The model's native
weight loader maps them to rank-local runtime parameters, including TP shards and
packed parameters.
"""
from collections.abc import Iterable
from concurrent.futures import ThreadPoolExecutor
from contextlib import nullcontext
from dataclasses import asdict, dataclass
from typing import TYPE_CHECKING, ClassVar, Self
import torch
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
TrainerInitInfo,
TrainerWeightTransferEngine,
VLLMWeightSyncClient,
WeightSource,
WeightTransferEngine,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
worker_init_payload,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_common import (
trainer_init as open_trainer_endpoint,
)
from vllm.model_executor.model_loader.checkpoint_weight_patch import (
CheckpointWeightPatch,
load_checkpoint_weight_patches,
)
__all__ = [
"SparseWeightPatch",
"SparseNCCLTrainerInitInfo",
"SparseNCCLWeightTransferUpdateInfo",
"SparseNCCLWeightTransferEngine",
"SparseNCCLTrainerWeightTransferEngine",
]
@dataclass
class SparseWeightPatch:
"""A sparse patch in checkpoint coordinates."""
name: str
indices: torch.Tensor
values: torch.Tensor
full_shape: tuple[int, ...]
"""Full checkpoint shape."""
@dataclass
class SparseNCCLTrainerInitInfo(TrainerInitInfo):
"""Trainer-side init info for the sparse NCCL weight transfer backend.
Same rendezvous shape as the dense NCCL backend (the sender opens its
endpoint as NCCL rank 0), but with no packed wire params: sparse transfers
are never packed. `backend` is the factory dispatch key."""
backend: ClassVar[str] = "sparse_nccl"
master_address: str
master_port: int
world_size: int
@dataclass
class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Update info for the sparse NCCL weight transfer backend."""
names: list[str]
dtype_names: list[str]
shapes: list[list[int]]
num_updates_list: list[int]
"""Number of sparse entries to receive for each parameter in ``names``."""
def __post_init__(self) -> None:
num_params = len(self.names)
if len(self.dtype_names) != num_params:
raise ValueError(
f"`dtype_names` should be of the same size as `names`: "
f"got {len(self.dtype_names)} and {len(self.names)}"
)
if len(self.shapes) != num_params:
raise ValueError(
f"`shapes` should be of the same size as `names`: "
f"got {len(self.shapes)} and {len(self.names)}"
)
if len(self.num_updates_list) == 0:
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
if len(self.num_updates_list) != num_params:
raise ValueError(
f"`num_updates_list` should be of the same size as `names`: "
f"got {len(self.num_updates_list)} and {len(self.names)}"
)
if any(num_updates < 0 for num_updates in self.num_updates_list):
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
class SparseNCCLWeightTransferEngine(
WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo]
):
"""Sparse weight transfer engine using NCCL.
Receives checkpoint-coordinate patches broadcast from the trainer and applies
them through the model's native weight loader. Sparse updates modify initialized
model tensors in place, so the layerwise reload lifecycle is not used.
"""
init_info_cls = NCCLWeightTransferInitInfo
update_info_cls = SparseNCCLWeightTransferUpdateInfo
supports_draft_weight_update = False
def __init__(
self,
config: WeightTransferConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
super().__init__(config, vllm_config, device, model)
self.model_update_group: PyNcclCommunicator | None = None
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
"""Initialize the NCCL process group with the trainer."""
self.model_update_group = worker_init_process_group(
init_info, self.parallel_config
)
def start_weight_update(self) -> None:
"""No-op: sparse patches are applied in place, no layerwise reload."""
pass
def finish_weight_update(self) -> None:
"""No-op: sparse patches are applied in place, no layerwise reload."""
pass
def receive_weights(self, update_info: SparseNCCLWeightTransferUpdateInfo) -> None:
"""Receive sparse flat-index patches from the trainer and apply them."""
if self.model_update_group is None:
raise RuntimeError(
"NCCL weight transfer not initialized. "
"Call init_transfer_engine() first."
)
# Receive on the communicator's device rather than relying on the
# ambient current device in the RPC thread.
device = self.model_update_group.device
with torch.accelerator.device_index(device.index):
stream = torch.cuda.current_stream(device=device)
patches = []
for name, dtype_name, shape, num_updates in zip(
update_info.names,
update_info.dtype_names,
update_info.shapes,
update_info.num_updates_list,
strict=True,
):
dtype = getattr(torch, dtype_name)
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
values = torch.empty(num_updates, dtype=dtype, device=device)
self.model_update_group.broadcast(indices, src=0, stream=stream)
self.model_update_group.broadcast(values, src=0, stream=stream)
patches.append(
CheckpointWeightPatch(
name=name,
shape=tuple(shape),
dtype=dtype,
values=values,
indices=indices,
)
)
load_checkpoint_weight_patches(self.model, patches)
def shutdown(self) -> None:
if self.model_update_group is not None:
self.model_update_group.destroy()
self.model_update_group = None
class SparseNCCLTrainerWeightTransferEngine(
TrainerWeightTransferEngine[SparseNCCLTrainerInitInfo]
):
"""Trainer-side sparse NCCL weight transfer engine.
Broadcasts flat-index (indices, values) patches from NCCL rank 0 while the
inference-side `update_weights` runs concurrently on a side thread (the
worker's recvs rendezvous inside the same NCCL broadcasts). `send_weights`
owns a complete one-shot lifecycle. RL infrastructure that owns the generic
client lifecycle can call `send_weight_chunk` between one start and finish.
Sparse patches differ every round, so they are not a stable `WeightSource`:
the engine takes no `source`, and patches are passed directly to the send
methods. An empty patch list is a no-op.
Only the designated trainer sender joins the transfer group; other trainer
ranks skip sparse sends.
"""
init_info_cls = SparseNCCLTrainerInitInfo
def __init__(
self,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
is_sender: bool = True,
) -> None:
# Sparse is a delta backend: it takes per-round patches via
# send_weights, so a `source` would silently never be sent. The
# parameter exists only to match the base/factory signature.
if source is not None:
raise ValueError(
"Sparse NCCL weight transfer takes no WeightSource; pass each "
"round's patches to send_weights(patches) instead."
)
super().__init__(client=client, source=source, is_sender=is_sender)
self.model_update_group: PyNcclCommunicator | None = None
@classmethod
def trainer_init(
cls,
init_info: SparseNCCLTrainerInitInfo,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
) -> Self:
engine = cls(client=client, source=source, is_sender=init_info.is_sender)
if not engine.is_sender:
return engine
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
# Sparse transfers are never packed, so the worker keeps the unpacked
# defaults on its init info.
worker_init_info = NCCLWeightTransferInitInfo(
master_address=init_info.master_address,
master_port=init_info.master_port,
rank_offset=1,
world_size=init_info.world_size,
)
# The inference workers block inside init_weight_transfer_engine waiting
# for the NCCL rendezvous, so we kick that off on a side thread while we
# open the trainer endpoint (rank 0); both sides must rendezvous together.
with ThreadPoolExecutor(max_workers=1) as exe:
future = exe.submit(
engine.client.init_weight_transfer_engine,
worker_init_payload(worker_init_info),
)
# Open the trainer endpoint as NCCL rank 0 on the current device
# (the init info satisfies the helper's rendezvous protocol).
engine.model_update_group = open_trainer_endpoint(init_info)
future.result() # surface any inference-side init error
return engine
def send_weights(self, patches: Iterable[SparseWeightPatch] | None = None) -> None:
"""Broadcast one sparse update through a one-shot lifecycle."""
patches = self._prepare_patches(patches)
if not patches:
return
self.client.start_weight_update()
self._broadcast_chunk(patches)
self.client.finish_weight_update()
def send_weight_chunk(
self, patches: Iterable[SparseWeightPatch] | None = None
) -> None:
"""Broadcast one chunk inside a caller-owned weight update lifecycle."""
patches = self._prepare_patches(patches)
if not patches:
return
self._broadcast_chunk(patches)
def _prepare_patches(
self, patches: Iterable[SparseWeightPatch] | None
) -> list[SparseWeightPatch]:
if not self.is_sender:
return []
prepared = list(patches) if patches is not None else []
if not prepared:
return []
if self.model_update_group is None:
raise RuntimeError("trainer_init() must be called before sending weights.")
for patch in prepared:
self._validate_patch(patch)
return prepared
def _broadcast_chunk(self, patches: list[SparseWeightPatch]) -> None:
assert self.model_update_group is not None
device = self.model_update_group.device
device_context = (
torch.accelerator.device_index(device.index)
if device.type != "cpu"
else nullcontext()
)
with device_context:
update_info = SparseNCCLWeightTransferUpdateInfo(
names=[patch.name for patch in patches],
dtype_names=[
str(patch.values.dtype).split(".")[-1] for patch in patches
],
shapes=[list(patch.full_shape) for patch in patches],
num_updates_list=[patch.indices.numel() for patch in patches],
)
# update_weights (workers receive) must run concurrently with the
# trainer-side broadcasts — both rendezvous inside the same NCCL calls.
executor = ThreadPoolExecutor(max_workers=1)
try:
future = executor.submit(
self.client.update_weights, asdict(update_info)
)
# Surface an RPC that failed before the worker entered NCCL rather
# than broadcasting to a peer that will never receive.
if future.done():
future.result()
stream = torch.cuda.current_stream()
for patch in patches:
self.model_update_group.broadcast(
patch.indices, src=0, stream=stream
)
self.model_update_group.broadcast(
patch.values, src=0, stream=stream
)
future.result() # surface inference-side errors
finally:
# A failed broadcast can leave the RPC blocked in the matching
# receive. Waiting for that thread would hide the original error.
executor.shutdown(wait=False, cancel_futures=True)
self._post_send_sync()
@staticmethod
def _validate_patch(patch: SparseWeightPatch) -> None:
"""Reject a malformed patch before starting the NCCL transfer."""
if patch.full_shape is None:
raise ValueError(f"Sparse patch requires full_shape: {patch.name}")
if patch.indices.dtype != torch.int32:
raise ValueError(
f"Sparse weight updates require int32 indices: {patch.name}"
)
if patch.indices.ndim != 1 or patch.values.ndim != 1:
raise ValueError(
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
)
if patch.indices.numel() != patch.values.numel():
raise ValueError(
f"`indices` and `values` must have matching lengths for {patch.name}"
)
def _post_send_sync(self) -> None:
"""Wait for the broadcasts to land before returning, so a caller may
rebuild or free the patch tensors as soon as a send method returns rather
than relying on same-stream ordering. See
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
cross-rank barrier."""
if torch.cuda.is_available():
torch.cuda.current_stream().synchronize()
def shutdown(self) -> None:
if self.model_update_group is not None:
self.model_update_group.destroy()
self.model_update_group = None