[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers
- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs, moriio, hisparse, example and hidden-states connectors - Reduce weight_transfer to the base API used by model loading - Remove kv_offload tiering, simple_kv_offload, ec cpu/example, snapshot command, engine snapshot server - Remove transformers model fallback, dsv4 tokenizers, dead platforms cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn ops, dead layers (short_conv, fused_embed_norm, indexer_topk, relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar - Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant - Rebuild parser hubs around Qwen3 adapters only (36 dead adapters) - Remove dead tests with the code they covered Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,459 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""IPC-based weight transfer engine using CUDA IPC for communication."""
|
||||
|
||||
import pickle
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Self
|
||||
|
||||
import pybase64 as base64
|
||||
import torch
|
||||
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
TrainerInitInfo,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
WeightTransferEngine,
|
||||
WeightTransferInitInfo,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
PackedBufferImporter,
|
||||
packed_ipc_consumer,
|
||||
packed_ipc_producer,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class IPCWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Worker-side init info for IPC weight transfer. No rendezvous needed.
|
||||
|
||||
`packed` is a must-agree wire param: the trainer ships it here at the init
|
||||
handshake so the worker decodes with the same setting the trainer encoded
|
||||
with. The consumer rebuilds from the IPC handle + `tensor_sizes`, so it does
|
||||
not need the buffer size (producer-only)."""
|
||||
|
||||
packed: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class IPCTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for IPC weight transfer. No rendezvous needed;
|
||||
`rank` (from `TrainerInitInfo`) identifies this trainer process — rank 0
|
||||
ships the merged IPC handles. All ranks still join the handle all-gather.
|
||||
|
||||
`packed` / `packed_buffer_size_bytes` are the transfer's wire params. The
|
||||
trainer propagates `packed` to the worker at `trainer_init` so the two sides
|
||||
cannot disagree. `backend` is the factory dispatch key."""
|
||||
|
||||
backend: ClassVar[str] = "ipc"
|
||||
packed: bool = False
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
|
||||
|
||||
@dataclass
|
||||
class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Per-round update info for the IPC weight transfer backend."""
|
||||
|
||||
names: list[str]
|
||||
dtype_names: list[str]
|
||||
shapes: list[list[int]]
|
||||
ipc_handles: list[dict[str, tuple]] | dict[str, tuple] | None = None
|
||||
"""IPC handles mapping physical GPU UUID to rebuild_cuda_tensor args.
|
||||
For non-packed mode: list of per-parameter handle dicts.
|
||||
For packed mode: single handle dict for the packed buffer."""
|
||||
ipc_handles_pickled: str | None = None
|
||||
"""Base64-encoded pickled IPC handles, used for HTTP transport."""
|
||||
tensor_sizes: list[int] | None = None
|
||||
"""Per-parameter sizes in bytes within the packed buffer.
|
||||
Required when packed=True, unused otherwise."""
|
||||
|
||||
def __post_init__(self):
|
||||
if self.ipc_handles_pickled is not None:
|
||||
if self.ipc_handles is not None:
|
||||
raise ValueError(
|
||||
"Cannot specify both `ipc_handles` and `ipc_handles_pickled`"
|
||||
)
|
||||
|
||||
if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION:
|
||||
raise ValueError(
|
||||
"Refusing to deserialize `ipc_handles_pickled` without "
|
||||
"VLLM_ALLOW_INSECURE_SERIALIZATION=1"
|
||||
)
|
||||
|
||||
self.ipc_handles = pickle.loads(base64.b64decode(self.ipc_handles_pickled))
|
||||
self.ipc_handles_pickled = None
|
||||
|
||||
if self.ipc_handles is None:
|
||||
raise ValueError(
|
||||
"Either `ipc_handles` or `ipc_handles_pickled` must be provided"
|
||||
)
|
||||
num_params = len(self.names)
|
||||
if len(self.dtype_names) != num_params:
|
||||
raise ValueError(
|
||||
f"`dtype_names` should be of the same size as `names`: "
|
||||
f"got {len(self.dtype_names)} and {len(self.names)}"
|
||||
)
|
||||
if len(self.shapes) != num_params:
|
||||
raise ValueError(
|
||||
f"`shapes` should be of the same size as `names`: "
|
||||
f"got {len(self.shapes)} and {len(self.names)}"
|
||||
)
|
||||
# Unpacked transfers carry a per-parameter handle dict (a list); packed
|
||||
# transfers carry a single dict for the whole buffer, so the list check
|
||||
# only applies to the list case.
|
||||
if isinstance(self.ipc_handles, list) and len(self.ipc_handles) != num_params:
|
||||
raise ValueError(
|
||||
f"`ipc_handles` should be of the same size as `names`: "
|
||||
f"got {len(self.ipc_handles)} and {len(self.names)}"
|
||||
)
|
||||
|
||||
|
||||
class IPCWeightTransferEngine(
|
||||
WeightTransferEngine[IPCWeightTransferInitInfo, IPCWeightTransferUpdateInfo]
|
||||
):
|
||||
"""Weight transfer engine using CUDA IPC between trainer and workers.
|
||||
|
||||
This implementation uses CUDA IPC to transfer weights from the trainer (rank 0)
|
||||
to all inference workers in a process group. IPC handles are used to share
|
||||
memory between processes on the same node.
|
||||
"""
|
||||
|
||||
# Define backend-specific dataclass types
|
||||
init_info_cls = IPCWeightTransferInitInfo
|
||||
update_info_cls = IPCWeightTransferUpdateInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
# Set from the trainer-supplied init info at the handshake; defaults are
|
||||
# only for the (unreachable) receive-before-init case.
|
||||
self.packed = False
|
||||
# Shared across all chunks of every packed transfer this engine
|
||||
# receives; see PackedBufferImporter for the refcount contract.
|
||||
self._packed_importer = PackedBufferImporter()
|
||||
|
||||
def init_transfer_engine(self, init_info: IPCWeightTransferInitInfo) -> None:
|
||||
"""Initialize the weight transfer mechanism. No data-plane rendezvous is
|
||||
needed for IPC; this just records the trainer-supplied wire params so the
|
||||
worker decodes exactly as the trainer encoded.
|
||||
|
||||
Args:
|
||||
init_info: IPC initialization info (carries `packed`).
|
||||
|
||||
"""
|
||||
self.packed = init_info.packed
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Initialize layerwise reloading for the incoming checkpoint weights."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
|
||||
initialize_layerwise_reload(self.model)
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Finalize layerwise reloading after all weights have been received."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
)
|
||||
|
||||
finalize_layerwise_reload(self.model, self.model_config)
|
||||
# Every reduce_tensor call is a fresh export with its own refcount
|
||||
# slot, so releasing once per update always balances this update's
|
||||
# export and lets the trainer reclaim its staging buffer. Callers
|
||||
# that skip finish are still covered by the replace-on-next-export
|
||||
# path inside the importer.
|
||||
self._packed_importer.close()
|
||||
|
||||
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
|
||||
"""Receive weights from the trainer via CUDA IPC handles and load them.
|
||||
|
||||
Whether the transfer is packed is read from `self.packed`, set at the
|
||||
init handshake from the trainer's init info, so it is guaranteed to
|
||||
match how the trainer encoded.
|
||||
|
||||
Args:
|
||||
update_info: IPC update info containing parameter names, dtypes, shapes,
|
||||
and IPC handles. Each IPC handle is a mapping between physical
|
||||
GPU UUID and the rebuild_cuda_tensor args tuple.
|
||||
|
||||
"""
|
||||
# Use the worker's assigned device rather than the ambient current
|
||||
# device: the receive path is no longer wrapped in
|
||||
# `with torch.device(self.device)` by the caller, so the current device
|
||||
# is not guaranteed to match self.device. The IPC tensors must be
|
||||
# rebuilt on the device the model lives on.
|
||||
device_index = self.device.index
|
||||
|
||||
if self.packed:
|
||||
if update_info.tensor_sizes is None:
|
||||
raise ValueError("`tensor_sizes` is required when packed=True")
|
||||
assert isinstance(update_info.ipc_handles, dict)
|
||||
weights = packed_ipc_consumer(
|
||||
ipc_handle=update_info.ipc_handles,
|
||||
names=update_info.names,
|
||||
shapes=update_info.shapes,
|
||||
dtype_names=update_info.dtype_names,
|
||||
tensor_sizes=update_info.tensor_sizes,
|
||||
device_index=device_index,
|
||||
importer=self._packed_importer,
|
||||
)
|
||||
else:
|
||||
assert isinstance(update_info.ipc_handles, list)
|
||||
weights = []
|
||||
for name, ipc_handle in zip(
|
||||
update_info.names,
|
||||
update_info.ipc_handles,
|
||||
):
|
||||
props = torch.cuda.get_device_properties(device_index)
|
||||
physical_gpu_id = str(props.uuid)
|
||||
|
||||
if physical_gpu_id not in ipc_handle:
|
||||
raise ValueError(
|
||||
f"IPC handle not found for GPU UUID "
|
||||
f"{physical_gpu_id}. "
|
||||
f"Available UUIDs: {list(ipc_handle.keys())}"
|
||||
)
|
||||
|
||||
args = ipc_handle[physical_gpu_id]
|
||||
list_args = list(args)
|
||||
# Index 6 of the args from reduce_tensor is the device_index.
|
||||
# We need to overwrite it with the receiver's device index.
|
||||
list_args[6] = device_index
|
||||
weight = rebuild_cuda_tensor(*list_args)
|
||||
weights.append((name, weight))
|
||||
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
disable_mtp_completeness_check,
|
||||
)
|
||||
|
||||
with disable_mtp_completeness_check():
|
||||
self.model.load_weights(weights)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._packed_importer.close()
|
||||
|
||||
|
||||
class IPCTrainerWeightTransferEngine(TrainerWeightTransferEngine[IPCTrainerInitInfo]):
|
||||
"""Trainer-side CUDA IPC weight transfer engine.
|
||||
|
||||
Called on every trainer rank. For multi-rank (e.g. FSDP) trainers all ranks
|
||||
iterate the source (materializing each tensor) and contribute to the
|
||||
IPC-handle all-gather; only the sender (rank 0) ships the merged handles to
|
||||
the inference side. IPC transfer is straight-line (no concurrent broadcast
|
||||
like NCCL): `update_weights` *is* the transfer, and it rides the client, so
|
||||
it no-ops on non-senders.
|
||||
|
||||
`packed` / `packed_buffer_size_bytes` come from `IPCTrainerInitInfo`; the
|
||||
sender propagates `packed` to the worker at `trainer_init`.
|
||||
"""
|
||||
|
||||
init_info_cls = IPCTrainerInitInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource,
|
||||
is_sender: bool = True,
|
||||
packed: bool = False,
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
) -> None:
|
||||
super().__init__(client=client, source=source, is_sender=is_sender)
|
||||
self.packed = packed
|
||||
self.packed_buffer_size_bytes = packed_buffer_size_bytes
|
||||
self.device_index = torch.accelerator.current_device_index()
|
||||
self.gpu_uuid = str(torch.cuda.get_device_properties(self.device_index).uuid)
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
init_info: IPCTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
if source is None:
|
||||
raise ValueError("IPC trainer weight transfer requires a WeightSource.")
|
||||
engine = cls(
|
||||
client=client,
|
||||
source=source,
|
||||
is_sender=init_info.is_sender,
|
||||
packed=init_info.packed,
|
||||
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
|
||||
)
|
||||
# IPC needs no data-plane rendezvous. The sender ships the must-agree
|
||||
# `packed` flag so the worker decodes exactly as this trainer encodes.
|
||||
if engine.is_sender:
|
||||
engine.client.init_weight_transfer_engine({"packed": init_info.packed})
|
||||
return engine
|
||||
|
||||
def send_weights(self) -> None:
|
||||
assert self.source is not None
|
||||
source = self.source
|
||||
if self.is_sender:
|
||||
self.client.start_weight_update()
|
||||
# Unpacked returns strong refs to its IPC-shared copies; they must stay
|
||||
# alive across `finish_weight_update` too.
|
||||
weight_refs = self._send(source)
|
||||
if self.is_sender:
|
||||
self.client.finish_weight_update()
|
||||
self._post_send_sync()
|
||||
del weight_refs
|
||||
|
||||
# ---- data plane (runs on all ranks; only the sender ships) ----
|
||||
|
||||
def _send(self, source: WeightSource) -> list[torch.Tensor] | None:
|
||||
if self.packed:
|
||||
self._send_packed(source)
|
||||
return None
|
||||
return self._send_unpacked(source)
|
||||
|
||||
def _all_gather_and_merge_handles(
|
||||
self,
|
||||
handles: list[dict[str, tuple]],
|
||||
) -> list[dict[str, tuple]]:
|
||||
"""All-gather and merge IPC handle dicts across ranks in one call.
|
||||
|
||||
Each rank contributes a list of {gpu_uuid: ipc_args} dicts (one
|
||||
per parameter or one per chunk). A single all_gather_object
|
||||
collects every rank's full list, then the sender merges per-index so
|
||||
each dict maps every GPU UUID to its args.
|
||||
|
||||
The all-gather runs over the *default* process group; this assumes the
|
||||
default group is exactly the set of colocated trainer ranks and that the
|
||||
sender is a member. No-op (returns handles unchanged) when no distributed
|
||||
group exists.
|
||||
"""
|
||||
if (
|
||||
not torch.distributed.is_initialized()
|
||||
or torch.distributed.get_world_size() == 1
|
||||
):
|
||||
return handles
|
||||
|
||||
world_size = torch.distributed.get_world_size()
|
||||
gathered: list[list[dict[str, tuple]] | None] = [None] * world_size
|
||||
torch.distributed.all_gather_object(gathered, handles)
|
||||
torch.distributed.barrier()
|
||||
torch.cuda.synchronize()
|
||||
|
||||
if self.is_sender:
|
||||
merged: list[dict[str, tuple]] = []
|
||||
for param_idx in range(len(handles)):
|
||||
m: dict[str, tuple] = {}
|
||||
for rank_handles in gathered:
|
||||
if rank_handles is not None:
|
||||
m.update(rank_handles[param_idx])
|
||||
merged.append(m)
|
||||
return merged
|
||||
return [{} for _ in handles]
|
||||
|
||||
@staticmethod
|
||||
def _post_send_sync() -> None:
|
||||
"""Barrier + ipc_collect after a send; no-op if single-GPU."""
|
||||
if (
|
||||
torch.distributed.is_initialized()
|
||||
and torch.distributed.get_world_size() > 1
|
||||
):
|
||||
torch.distributed.barrier()
|
||||
torch.cuda.ipc_collect()
|
||||
|
||||
def _send_unpacked(self, source: WeightSource) -> list[torch.Tensor]:
|
||||
"""Iterate the source, build one IPC handle per param, all-gather the
|
||||
handles across ranks, and (sender) ship them in one update call.
|
||||
|
||||
Returns the strong refs to every contiguous copy. reduce_tensor's args
|
||||
do NOT keep storage alive, and non-contiguous inputs allocate fresh
|
||||
storage in .contiguous(); the caller must keep these alive until the
|
||||
post-send barrier (past `finish`) so the consumer's IPC views stay valid.
|
||||
"""
|
||||
names: list[str] = []
|
||||
dtype_names: list[str] = []
|
||||
shapes: list[list[int]] = []
|
||||
ipc_handles: list[dict[str, tuple]] = []
|
||||
weight_refs: list[torch.Tensor] = []
|
||||
|
||||
for name, tensor in source:
|
||||
# FSDP shards were gathered by the source; ensure contiguity here.
|
||||
names.append(name)
|
||||
dtype_names.append(str(tensor.dtype).split(".")[-1])
|
||||
shapes.append(list(tensor.shape))
|
||||
|
||||
weight = tensor.detach().contiguous()
|
||||
weight_refs.append(weight)
|
||||
_, ipc_args = reduce_tensor(weight)
|
||||
ipc_handles.append({self.gpu_uuid: ipc_args})
|
||||
|
||||
ipc_handles = self._all_gather_and_merge_handles(ipc_handles)
|
||||
self._do_send(
|
||||
names=names,
|
||||
dtype_names=dtype_names,
|
||||
shapes=shapes,
|
||||
ipc_handles=ipc_handles,
|
||||
)
|
||||
return weight_refs
|
||||
|
||||
def _send_packed(self, source: WeightSource) -> None:
|
||||
"""Send weights in bounded-memory chunks (packed mode)."""
|
||||
for chunk in packed_ipc_producer(
|
||||
iterator=iter(source),
|
||||
gpu_uuid=self.gpu_uuid,
|
||||
post_iter_func=lambda item: item[1],
|
||||
buffer_size_bytes=self.packed_buffer_size_bytes,
|
||||
):
|
||||
ipc_handle = self._all_gather_and_merge_handles([chunk.ipc_handle])[0]
|
||||
self._do_send(
|
||||
names=chunk.names,
|
||||
dtype_names=chunk.dtype_names,
|
||||
shapes=chunk.shapes,
|
||||
ipc_handles=ipc_handle,
|
||||
tensor_sizes=chunk.tensor_sizes,
|
||||
)
|
||||
# Per-chunk barrier: the producer reuses a single IPC buffer across
|
||||
# chunks, but only the sender waits for the consumers (via _do_send).
|
||||
# Without syncing every rank here, non-sender ranks race ahead and
|
||||
# overwrite their buffer while their colocated worker is still
|
||||
# reading the current chunk, silently corrupting the transfer.
|
||||
self._post_send_sync()
|
||||
|
||||
def _do_send(
|
||||
self,
|
||||
names: list[str],
|
||||
dtype_names: list[str],
|
||||
shapes: list[list[int]],
|
||||
ipc_handles: list[dict[str, tuple]] | dict[str, tuple],
|
||||
tensor_sizes: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Build one update payload and ship it via the client. Only the sender
|
||||
ships (non-sender ranks already contributed to the handle all-gather).
|
||||
|
||||
Emits raw `ipc_handles`; transports that cannot carry them natively
|
||||
(HTTP/JSON) pickle them in their client (see `HTTPVLLMWeightSyncClient`).
|
||||
"""
|
||||
if not self.is_sender:
|
||||
return
|
||||
update_fields: dict[str, Any] = {
|
||||
"names": names,
|
||||
"dtype_names": dtype_names,
|
||||
"shapes": shapes,
|
||||
"ipc_handles": ipc_handles,
|
||||
}
|
||||
if tensor_sizes is not None:
|
||||
update_fields["tensor_sizes"] = tensor_sizes
|
||||
|
||||
update_info = IPCWeightTransferUpdateInfo(**update_fields)
|
||||
self.client.update_weights(asdict(update_info))
|
||||
@@ -1,864 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared helpers for the NCCL M2N (`nccl_m2n`) weight transfer backend.
|
||||
|
||||
M2N reshards a tensor between two disjoint meshes of ranks that live in one
|
||||
communicator: the trainer occupies ranks `[0, T)` and the inference workers
|
||||
`[T, T + N)`. This module holds everything both sides need — the optional
|
||||
runtime import, layout descriptors, and the conversion to `nccl.m2n` types —
|
||||
so the engine module stays about the transfer itself.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.base import ParamMeta
|
||||
from vllm.utils.nccl import unpinned_nccl_env
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
# Placement code for a mesh axis that replicates. Any other (non-negative) code
|
||||
# is the tensor dim that axis shards. Ints rather than `nccl.m2n` objects so
|
||||
# layouts survive the JSON init handshake without a custom encoder.
|
||||
REPLICATE = -1
|
||||
|
||||
# `NCCL_RESHARD_MAX_TENSOR_DIMS`; 4-D and higher are rejected by the library.
|
||||
MAX_TENSOR_DIMS = 3
|
||||
|
||||
MESH_NDIMS = 2
|
||||
|
||||
# Destination mesh convention. Keeping these roles named avoids scattering
|
||||
# positional axis assumptions through the worker-side planner.
|
||||
DESTINATION_REPLICA_AXIS = 0
|
||||
DESTINATION_SHARD_AXIS = 1
|
||||
|
||||
# Conservative capacities accepted by the supported nccl_m2n algorithms. They
|
||||
# are not exposed by its Python API yet; remove these mirrors once nccl_m2n
|
||||
# provides communication-free plan validation.
|
||||
MAX_SOURCE_SHARDS = 16
|
||||
MAX_DEST_SHARDS = 64
|
||||
|
||||
# Increment when the JSON init payload changes incompatibly.
|
||||
M2N_WIRE_SCHEMA_VERSION = 2
|
||||
|
||||
SUPPORTED_DTYPES: frozenset[torch.dtype] = frozenset(
|
||||
dtype
|
||||
for dtype in (
|
||||
getattr(torch, name, None)
|
||||
for name in (
|
||||
"int8",
|
||||
"uint8",
|
||||
"float8_e4m3fn",
|
||||
"float8_e5m2",
|
||||
"float16",
|
||||
"bfloat16",
|
||||
"int32",
|
||||
"uint32",
|
||||
"float32",
|
||||
"int64",
|
||||
"uint64",
|
||||
"float64",
|
||||
)
|
||||
)
|
||||
if dtype is not None
|
||||
)
|
||||
|
||||
_IMPORT_HINT = (
|
||||
"The nccl_m2n weight transfer backend requires the `nccl-extensions` "
|
||||
"package (and its `nccl4py` dependency), which is not installed by vLLM. "
|
||||
"See https://github.com/NVIDIA/nccl-extensions for build "
|
||||
"and install instructions. It needs NCCL 2.30.5 or newer."
|
||||
)
|
||||
|
||||
|
||||
def prepare_m2n_local_runtime(m2n: Any, max_cta: int | None) -> tuple[Any, int, Any]:
|
||||
"""Prepare rank-local resources before any rank enters NCCL init."""
|
||||
import nccl.core as nccl
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
if not isinstance(device, int) or isinstance(device, bool) or device < 0:
|
||||
raise RuntimeError(f"invalid current accelerator device index: {device!r}")
|
||||
handle = m2n.Handle.create(m2n.Config(max_cta=max_cta))
|
||||
return nccl, device, handle
|
||||
|
||||
|
||||
def create_m2n_communicator(
|
||||
nccl: Any,
|
||||
group: "StatelessProcessGroup",
|
||||
unique_id_bytes: bytes | None,
|
||||
) -> Any:
|
||||
"""Create the M2N data-plane communicator through NCCL4Py."""
|
||||
if unique_id_bytes is None:
|
||||
envelope: dict[str, bytes | str | None] | None = None
|
||||
if group.rank == 0:
|
||||
try:
|
||||
envelope = {
|
||||
"unique_id": bytes(nccl.get_unique_id().as_bytes),
|
||||
"error": None,
|
||||
}
|
||||
except Exception as exc:
|
||||
envelope = {
|
||||
"unique_id": None,
|
||||
"error": f"{type(exc).__name__}: {exc}",
|
||||
}
|
||||
envelope = group.broadcast_obj(envelope, src=0)
|
||||
if not isinstance(envelope, Mapping) or set(envelope) != {"unique_id", "error"}:
|
||||
raise RuntimeError("invalid NCCL4Py unique-id envelope")
|
||||
error = envelope["error"]
|
||||
received_unique_id = envelope["unique_id"]
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
raise RuntimeError("invalid NCCL4Py unique-id error")
|
||||
raise RuntimeError(f"NCCL4Py unique-id generation failed: {error}")
|
||||
if not isinstance(received_unique_id, bytes):
|
||||
raise RuntimeError("NCCL4Py unique-id broadcast did not return bytes")
|
||||
unique_id_bytes = received_unique_id
|
||||
|
||||
unique_id = nccl.UniqueId.from_bytes(unique_id_bytes)
|
||||
with unpinned_nccl_env():
|
||||
return nccl.Communicator.init(
|
||||
nranks=group.world_size,
|
||||
rank=group.rank,
|
||||
unique_id=unique_id,
|
||||
)
|
||||
|
||||
|
||||
def import_m2n() -> Any:
|
||||
"""Import `nccl.m2n` lazily, with an actionable error when it is missing.
|
||||
|
||||
Deferred so that importing vLLM — or any other weight transfer backend —
|
||||
never requires the m2n runtime to be present.
|
||||
"""
|
||||
try:
|
||||
import nccl.m2n as m2n
|
||||
except ImportError as e:
|
||||
raise ImportError(f"{_IMPORT_HINT} (import failed: {e})") from e
|
||||
return m2n
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NMesh:
|
||||
"""One side's rank topology — pure topology, no tensor placement.
|
||||
|
||||
Mirrors `ncclMesh_t`: a 2-axis mesh owning the contiguous rank interval
|
||||
`[start_rank, start_rank + dims[0] * dims[1])`. There is no 1-D mesh; a
|
||||
single-axis topology is spelled with a second axis of size 1.
|
||||
|
||||
A model may use a different factorization for each tensor. For example,
|
||||
dense weights can use a DP x TP mesh while expert weights use EDP x EP.
|
||||
"""
|
||||
|
||||
dims: tuple[int, int]
|
||||
start_rank: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "dims", tuple(self.dims))
|
||||
if len(self.dims) != MESH_NDIMS or any(
|
||||
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
|
||||
for dim in self.dims
|
||||
):
|
||||
raise ValueError(f"mesh dims must be {MESH_NDIMS} positive ints")
|
||||
if (
|
||||
not isinstance(self.start_rank, int)
|
||||
or isinstance(self.start_rank, bool)
|
||||
or self.start_rank < 0
|
||||
):
|
||||
raise ValueError(
|
||||
f"mesh start_rank must be non-negative, got {self.start_rank}"
|
||||
)
|
||||
|
||||
@property
|
||||
def size(self) -> int:
|
||||
return self.dims[0] * self.dims[1]
|
||||
|
||||
|
||||
# A tensor's placement over its side's mesh: one code per mesh axis
|
||||
# (`REPLICATE`, or the tensor dim that axis shards), or `REPLICATED` for a
|
||||
# tensor every rank holds in full.
|
||||
Placements = tuple[int, int]
|
||||
REPLICATED: Placements | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NLayout:
|
||||
"""A tensor's mesh and placement on one side of a transfer."""
|
||||
|
||||
mesh: M2NMesh
|
||||
placements: Placements | None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.mesh, M2NMesh):
|
||||
raise TypeError("M2N layout mesh must be an M2NMesh")
|
||||
if self.placements is not None:
|
||||
object.__setattr__(self, "placements", tuple(self.placements))
|
||||
|
||||
|
||||
def check_placements(placements: Placements, context: str = "placements") -> None:
|
||||
"""Reject placement pairs nccl_m2n cannot express."""
|
||||
if len(placements) != MESH_NDIMS:
|
||||
raise ValueError(f"{context} must have {MESH_NDIMS} entries")
|
||||
if any(not isinstance(code, int) or isinstance(code, bool) for code in placements):
|
||||
raise TypeError(f"{context} must contain integer placement codes")
|
||||
invalid = [code for code in placements if code < REPLICATE]
|
||||
if invalid:
|
||||
raise ValueError(
|
||||
f"{context} contains invalid placement code {invalid[0]}; valid "
|
||||
f"codes are {REPLICATE} (Replicate) and non-negative tensor dimensions"
|
||||
)
|
||||
num_sharded = sum(code != REPLICATE for code in placements)
|
||||
if num_sharded == 0:
|
||||
raise ValueError(
|
||||
f"{context}: a fully replicated tensor is carried as REPLICATED, "
|
||||
"not as two REPLICATE axes"
|
||||
)
|
||||
if num_sharded == MESH_NDIMS:
|
||||
raise ValueError(
|
||||
f"{context}: nccl_m2n needs one REPLICATE mesh axis, but "
|
||||
f"{placements} shards both"
|
||||
)
|
||||
|
||||
|
||||
def resolve_layout(
|
||||
mesh: M2NMesh,
|
||||
placements: Placements | None,
|
||||
context: str = "placements",
|
||||
) -> tuple[M2NMesh, Placements]:
|
||||
"""Pair one tensor's placement with the mesh m2n should see for it.
|
||||
|
||||
A replicated tensor needs a size-1 mesh axis to carry a no-op shard, since
|
||||
m2n has no `{REPLICATE, REPLICATE}`. It is therefore described over the
|
||||
*same rank interval* re-factored as `(size, 1)`. That re-factoring is sound
|
||||
precisely because replication is order-independent — every rank holds the
|
||||
whole tensor, so it does not matter that `(a, b)` and `(size, 1)` walk the
|
||||
interval in a different order. A sharded tensor keeps its side's own
|
||||
factorization, where rank order decides who owns which shard.
|
||||
"""
|
||||
if placements is None:
|
||||
return M2NMesh((mesh.size, 1), mesh.start_rank), (REPLICATE, 0)
|
||||
check_placements(placements, context)
|
||||
return mesh, placements
|
||||
|
||||
|
||||
def validate_layout(
|
||||
mesh: M2NMesh,
|
||||
placements: Placements,
|
||||
shape: Sequence[int],
|
||||
context: str,
|
||||
) -> tuple[int, ...]:
|
||||
"""Validate one resolved layout and return its required local shape."""
|
||||
local_shape = list(shape)
|
||||
for axis, tensor_dim in enumerate(placements):
|
||||
if tensor_dim == REPLICATE:
|
||||
continue
|
||||
if tensor_dim >= len(shape):
|
||||
raise ValueError(
|
||||
f"{context} shards tensor dim {tensor_dim}, but parameter has "
|
||||
f"rank {len(shape)}"
|
||||
)
|
||||
factor = mesh.dims[axis]
|
||||
if shape[tensor_dim] % factor:
|
||||
raise ValueError(
|
||||
f"{context} shards dim {tensor_dim} (size {shape[tensor_dim]}) "
|
||||
f"over {factor} ranks, which does not divide evenly"
|
||||
)
|
||||
local_shape[tensor_dim] //= factor
|
||||
return tuple(local_shape)
|
||||
|
||||
|
||||
def expected_local_shape(meta: "M2NParamMeta") -> tuple[int, ...]:
|
||||
"""Validate a parameter's source layout and return its local shape."""
|
||||
mesh, placements = resolve_layout(
|
||||
meta.source_layout.mesh,
|
||||
meta.source_layout.placements,
|
||||
f"parameter '{meta.name}' source placements",
|
||||
)
|
||||
return validate_layout(
|
||||
mesh, placements, meta.shape, f"parameter '{meta.name}' source layout"
|
||||
)
|
||||
|
||||
|
||||
def check_transferable(name: str, dtype: torch.dtype, shape: Sequence[int]) -> None:
|
||||
"""Reject parameters nccl_m2n cannot transfer."""
|
||||
if dtype not in SUPPORTED_DTYPES:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' has dtype {dtype}, which nccl_m2n does not "
|
||||
f"support. Supported: {sorted(str(d) for d in SUPPORTED_DTYPES)}"
|
||||
)
|
||||
if not 1 <= len(shape) <= MAX_TENSOR_DIMS:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' has rank {len(shape)}; nccl_m2n supports "
|
||||
f"rank 1..{MAX_TENSOR_DIMS}"
|
||||
)
|
||||
|
||||
|
||||
def _shard_spec(mesh: M2NMesh, placements: Placements) -> tuple[int, int]:
|
||||
return next(
|
||||
(code, mesh.dims[axis])
|
||||
for axis, code in enumerate(placements)
|
||||
if code != REPLICATE
|
||||
)
|
||||
|
||||
|
||||
def _max_partition_overlaps(
|
||||
extent: int,
|
||||
partitions: int,
|
||||
other_partitions: int,
|
||||
) -> int:
|
||||
"""Return the maximum other partitions intersected by any one partition."""
|
||||
if extent == 0:
|
||||
return 0
|
||||
width = extent // partitions
|
||||
other_width = extent // other_partitions
|
||||
max_overlaps = 0
|
||||
for index in range(partitions):
|
||||
start = index * width
|
||||
end = start + width
|
||||
first_other = start // other_width
|
||||
last_other = (end - 1) // other_width
|
||||
max_overlaps = max(max_overlaps, last_other - first_other + 1)
|
||||
return max_overlaps
|
||||
|
||||
|
||||
def check_plan_limits(
|
||||
src: tuple[M2NMesh, Placements],
|
||||
dst: tuple[M2NMesh, Placements],
|
||||
name: str,
|
||||
shape: Sequence[int],
|
||||
) -> None:
|
||||
"""Reject plans whose per-shard fan-in or fan-out exceeds M2N capacity."""
|
||||
src_dim, src_shards = _shard_spec(*src)
|
||||
dst_dim, dst_shards = _shard_spec(*dst)
|
||||
if any(extent == 0 for extent in shape):
|
||||
max_sources = max_targets = 0
|
||||
elif src_dim != dst_dim:
|
||||
max_sources = src_shards
|
||||
max_targets = dst_shards
|
||||
else:
|
||||
extent = shape[src_dim]
|
||||
max_sources = _max_partition_overlaps(extent, dst_shards, src_shards)
|
||||
max_targets = _max_partition_overlaps(extent, src_shards, dst_shards)
|
||||
|
||||
if max_sources > MAX_SOURCE_SHARDS:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' would feed up to {max_sources} source shards "
|
||||
f"into one destination shard, over nccl_m2n MAX_SOURCES="
|
||||
f"{MAX_SOURCE_SHARDS}"
|
||||
)
|
||||
if max_targets > MAX_DEST_SHARDS:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' would fan one source shard out to up to "
|
||||
f"{max_targets} destination shards, over nccl_m2n "
|
||||
f"MAX_TARGETS={MAX_DEST_SHARDS}"
|
||||
)
|
||||
|
||||
|
||||
def validate_transfer_plan(
|
||||
metas: Sequence["M2NParamMeta"],
|
||||
dst_mesh: M2NMesh,
|
||||
dst_placements: Sequence[Placements | None],
|
||||
) -> None:
|
||||
"""Validate the complete ordered source-to-destination transfer plan."""
|
||||
if len(dst_placements) != len(metas):
|
||||
raise ValueError(
|
||||
f"destination plan has {len(dst_placements)} parameters, expected "
|
||||
f"{len(metas)}"
|
||||
)
|
||||
for meta, placement in zip(metas, dst_placements):
|
||||
check_transferable(meta.name, meta.dtype, meta.shape)
|
||||
src = resolve_layout(
|
||||
meta.source_layout.mesh,
|
||||
meta.source_layout.placements,
|
||||
f"parameter '{meta.name}' source placements",
|
||||
)
|
||||
validate_layout(*src, meta.shape, f"parameter '{meta.name}' source layout")
|
||||
dst = resolve_layout(
|
||||
dst_mesh,
|
||||
placement,
|
||||
f"parameter '{meta.name}' destination placements",
|
||||
)
|
||||
validate_layout(*dst, meta.shape, f"parameter '{meta.name}' destination layout")
|
||||
check_plan_limits(src, dst, meta.name, meta.shape)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NParamMeta(ParamMeta):
|
||||
"""`ParamMeta` extended with how the trainer places this tensor.
|
||||
|
||||
The base class carries only name / dtype / full shape, which is not enough
|
||||
to plan a reshard. `source_layout` carries both the per-parameter mesh and
|
||||
its placement on that mesh.
|
||||
"""
|
||||
|
||||
source_layout: M2NLayout
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.source_layout, M2NLayout):
|
||||
raise TypeError("source_layout must be an M2NLayout")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NWireParam:
|
||||
"""JSON-safe source metadata for one parameter in the init handshake."""
|
||||
|
||||
name: str
|
||||
dtype_name: str
|
||||
shape: tuple[int, ...]
|
||||
src_mesh_dims: tuple[int, int]
|
||||
src_placements: Placements | None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
shape = tuple(self.shape)
|
||||
dims = tuple(self.src_mesh_dims)
|
||||
placements = None if self.src_placements is None else tuple(self.src_placements)
|
||||
object.__setattr__(self, "shape", shape)
|
||||
object.__setattr__(self, "src_mesh_dims", dims)
|
||||
object.__setattr__(self, "src_placements", placements)
|
||||
if not isinstance(self.name, str) or not self.name:
|
||||
raise ValueError("M2N wire parameter name must not be empty")
|
||||
if not isinstance(self.dtype_name, str) or not self.dtype_name:
|
||||
raise ValueError(f"parameter '{self.name}' has an empty wire dtype name")
|
||||
if not shape or any(
|
||||
not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0
|
||||
for dim in shape
|
||||
):
|
||||
raise ValueError(
|
||||
f"parameter '{self.name}' shape must contain positive integers"
|
||||
)
|
||||
M2NMesh(cast(tuple[int, int], dims), 0)
|
||||
if placements is not None:
|
||||
check_placements(
|
||||
cast(Placements, placements),
|
||||
f"parameter '{self.name}' source placements",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: Mapping[str, Any]) -> "M2NWireParam":
|
||||
"""Parse one strict wire record from its JSON-decoded mapping."""
|
||||
fields = {
|
||||
"name",
|
||||
"dtype_name",
|
||||
"shape",
|
||||
"src_mesh_dims",
|
||||
"src_placements",
|
||||
}
|
||||
missing = fields - value.keys()
|
||||
extra = value.keys() - fields
|
||||
if missing or extra:
|
||||
details = []
|
||||
if missing:
|
||||
details.append(f"missing {sorted(missing)}")
|
||||
if extra:
|
||||
details.append(f"unexpected {sorted(extra)}")
|
||||
raise ValueError(f"invalid M2N wire parameter: {', '.join(details)}")
|
||||
return cls(**value)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Serialize this record using only JSON-compatible values."""
|
||||
return {
|
||||
"name": self.name,
|
||||
"dtype_name": self.dtype_name,
|
||||
"shape": list(self.shape),
|
||||
"src_mesh_dims": list(self.src_mesh_dims),
|
||||
"src_placements": (
|
||||
None if self.src_placements is None else list(self.src_placements)
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def source_plan_digest(params: Sequence[M2NWireParam]) -> str:
|
||||
"""Return a rank-independent digest of the ordered source wire plan."""
|
||||
payload = json.dumps(
|
||||
{
|
||||
"schema_version": M2N_WIRE_SCHEMA_VERSION,
|
||||
"params": [param.to_dict() for param in params],
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def complete_plan_digest(
|
||||
source_digest: str,
|
||||
dst_mesh: M2NMesh,
|
||||
dst_placements: Sequence[Placements | None],
|
||||
) -> str:
|
||||
"""Return a digest of the agreed source and destination plan."""
|
||||
if not isinstance(source_digest, str) or not source_digest:
|
||||
raise ValueError("source digest must be a non-empty string")
|
||||
payload = json.dumps(
|
||||
{
|
||||
"schema_version": M2N_WIRE_SCHEMA_VERSION,
|
||||
"source_digest": source_digest,
|
||||
"destination_mesh": {
|
||||
"dims": list(dst_mesh.dims),
|
||||
"start_rank": dst_mesh.start_rank,
|
||||
},
|
||||
"destination_placements": [
|
||||
None if placement is None else list(placement)
|
||||
for placement in dst_placements
|
||||
],
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def check_source_plan_agreement(
|
||||
group: "StatelessProcessGroup",
|
||||
params: Sequence[M2NWireParam],
|
||||
expected_digest: str | None = None,
|
||||
local_error: str | None = None,
|
||||
) -> str:
|
||||
"""Verify every participant entered with the same source plan."""
|
||||
prepared_error = local_error
|
||||
digest: str | None = None
|
||||
declared: str | None = expected_digest
|
||||
if prepared_error is None:
|
||||
try:
|
||||
digest = source_plan_digest(params)
|
||||
declared = digest if expected_digest is None else expected_digest
|
||||
if not isinstance(declared, str) or not declared:
|
||||
raise ValueError("declared source digest must be a non-empty string")
|
||||
except Exception as exc:
|
||||
prepared_error = f"{type(exc).__name__}: {exc}"
|
||||
elif not isinstance(prepared_error, str) or not prepared_error:
|
||||
prepared_error = "TypeError: local source error must be a non-empty string"
|
||||
|
||||
gathered = group.all_gather_obj(
|
||||
{
|
||||
"phase": "source",
|
||||
"digest": digest,
|
||||
"declared_digest": declared,
|
||||
"error": prepared_error,
|
||||
}
|
||||
)
|
||||
errors: list[str] = []
|
||||
expected_fields = {"phase", "digest", "declared_digest", "error"}
|
||||
if len(gathered) != group.world_size:
|
||||
errors.append(
|
||||
"control-plane all-gather returned "
|
||||
f"{len(gathered)} records for world size {group.world_size}"
|
||||
)
|
||||
agreed: list[tuple[int, str, str]] = []
|
||||
for rank, item in enumerate(gathered):
|
||||
if not isinstance(item, Mapping) or set(item) != expected_fields:
|
||||
errors.append(f"rank {rank}: invalid source preflight envelope")
|
||||
continue
|
||||
if item["phase"] != "source":
|
||||
errors.append(f"rank {rank}: invalid source preflight phase")
|
||||
continue
|
||||
error = item["error"]
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
errors.append(f"rank {rank}: invalid source preflight error")
|
||||
else:
|
||||
errors.append(f"rank {rank}: {error}")
|
||||
continue
|
||||
computed = item["digest"]
|
||||
claimed = item["declared_digest"]
|
||||
if (
|
||||
not isinstance(computed, str)
|
||||
or not computed
|
||||
or not isinstance(claimed, str)
|
||||
or not claimed
|
||||
):
|
||||
errors.append(f"rank {rank}: invalid source digest")
|
||||
continue
|
||||
agreed.append((rank, computed, claimed))
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
|
||||
)
|
||||
|
||||
canonical = agreed[0][1]
|
||||
for rank, computed, claimed in agreed:
|
||||
if computed != claimed:
|
||||
errors.append(
|
||||
f"rank {rank}: computed digest {computed} does not match "
|
||||
f"declared digest {claimed}"
|
||||
)
|
||||
if computed != canonical:
|
||||
errors.append(f"rank {rank}: source digest disagrees with rank 0")
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n source preflight rejected before NCCL init: " + "; ".join(errors)
|
||||
)
|
||||
return canonical
|
||||
|
||||
|
||||
def check_complete_plan_agreement(
|
||||
group: "StatelessProcessGroup",
|
||||
source_digest: str,
|
||||
dst_mesh: M2NMesh,
|
||||
dst_placements: Sequence[Placements | None],
|
||||
local_error: str | None = None,
|
||||
) -> str:
|
||||
"""Require every rank to accept the same complete transfer plan."""
|
||||
prepared_error = local_error
|
||||
digest: str | None = None
|
||||
if prepared_error is None:
|
||||
try:
|
||||
digest = complete_plan_digest(source_digest, dst_mesh, dst_placements)
|
||||
except Exception as exc:
|
||||
prepared_error = f"{type(exc).__name__}: {exc}"
|
||||
elif not isinstance(prepared_error, str) or not prepared_error:
|
||||
prepared_error = "TypeError: local plan error must be a non-empty string"
|
||||
|
||||
gathered = group.all_gather_obj(
|
||||
{"phase": "complete_plan", "digest": digest, "error": prepared_error}
|
||||
)
|
||||
errors: list[str] = []
|
||||
valid: list[tuple[int, str]] = []
|
||||
expected_fields = {"phase", "digest", "error"}
|
||||
if len(gathered) != group.world_size:
|
||||
errors.append(
|
||||
"control-plane all-gather returned "
|
||||
f"{len(gathered)} records for world size {group.world_size}"
|
||||
)
|
||||
for rank, item in enumerate(gathered):
|
||||
if (
|
||||
not isinstance(item, Mapping)
|
||||
or set(item) != expected_fields
|
||||
or item.get("phase") != "complete_plan"
|
||||
):
|
||||
errors.append(f"rank {rank}: invalid complete-plan envelope")
|
||||
continue
|
||||
error = item["error"]
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
errors.append(f"rank {rank}: invalid complete-plan error")
|
||||
else:
|
||||
errors.append(f"rank {rank}: {error}")
|
||||
continue
|
||||
rank_digest = item["digest"]
|
||||
if not isinstance(rank_digest, str) or not rank_digest:
|
||||
errors.append(f"rank {rank}: invalid complete-plan digest")
|
||||
continue
|
||||
valid.append((rank, rank_digest))
|
||||
if not errors:
|
||||
canonical = valid[0][1]
|
||||
for rank, rank_digest in valid[1:]:
|
||||
if rank_digest != canonical:
|
||||
errors.append(
|
||||
f"rank {rank}: complete-plan digest disagrees with rank 0"
|
||||
)
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n complete-plan preflight rejected before first reshard: "
|
||||
+ "; ".join(errors)
|
||||
)
|
||||
return valid[0][1]
|
||||
|
||||
|
||||
def check_runtime_ready_agreement(
|
||||
group: "StatelessProcessGroup", local_error: str | None
|
||||
) -> None:
|
||||
"""Require every rank to prepare local resources before NCCL init."""
|
||||
prepared_error = local_error
|
||||
if prepared_error is not None and (
|
||||
not isinstance(prepared_error, str) or not prepared_error
|
||||
):
|
||||
prepared_error = "TypeError: local runtime error must be a non-empty string"
|
||||
gathered = group.all_gather_obj({"phase": "runtime_ready", "error": prepared_error})
|
||||
errors: list[str] = []
|
||||
if len(gathered) != group.world_size:
|
||||
errors.append(
|
||||
"control-plane all-gather returned "
|
||||
f"{len(gathered)} records for world size {group.world_size}"
|
||||
)
|
||||
for rank, item in enumerate(gathered):
|
||||
if (
|
||||
not isinstance(item, Mapping)
|
||||
or set(item) != {"phase", "error"}
|
||||
or item.get("phase") != "runtime_ready"
|
||||
):
|
||||
errors.append(f"rank {rank}: invalid runtime readiness envelope")
|
||||
continue
|
||||
error = item.get("error")
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
errors.append(f"rank {rank}: invalid runtime readiness error")
|
||||
else:
|
||||
errors.append(f"rank {rank}: {error}")
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n local runtime preflight failed before NCCL init: "
|
||||
+ "; ".join(errors)
|
||||
)
|
||||
|
||||
|
||||
def check_data_plane_agreement(
|
||||
group: "StatelessProcessGroup",
|
||||
unique_id_bytes: bytes | None,
|
||||
local_error: str | None = None,
|
||||
) -> None:
|
||||
"""Agree on the NCCL rendezvous identity before NCCL init."""
|
||||
prepared_error = local_error
|
||||
uid_digest: str | None = None
|
||||
if prepared_error is None:
|
||||
try:
|
||||
if unique_id_bytes is not None:
|
||||
if not isinstance(unique_id_bytes, bytes):
|
||||
raise TypeError("NCCL unique id must decode to bytes")
|
||||
uid_digest = hashlib.sha256(unique_id_bytes).hexdigest()
|
||||
except Exception as exc:
|
||||
prepared_error = f"{type(exc).__name__}: {exc}"
|
||||
elif not isinstance(prepared_error, str) or not prepared_error:
|
||||
prepared_error = "TypeError: local data-plane error must be non-empty"
|
||||
|
||||
gathered = group.all_gather_obj(
|
||||
{
|
||||
"phase": "data_plane",
|
||||
"mode": "uid" if unique_id_bytes is not None else "tcp",
|
||||
"uid_digest": uid_digest,
|
||||
"error": prepared_error,
|
||||
}
|
||||
)
|
||||
expected_fields = {
|
||||
"phase",
|
||||
"mode",
|
||||
"uid_digest",
|
||||
"error",
|
||||
}
|
||||
errors: list[str] = []
|
||||
valid: list[tuple[int, str, str | None]] = []
|
||||
if len(gathered) != group.world_size:
|
||||
errors.append(
|
||||
"control-plane all-gather returned "
|
||||
f"{len(gathered)} records for world size {group.world_size}"
|
||||
)
|
||||
for rank, item in enumerate(gathered):
|
||||
if (
|
||||
not isinstance(item, Mapping)
|
||||
or set(item) != expected_fields
|
||||
or item.get("phase") != "data_plane"
|
||||
):
|
||||
errors.append(f"rank {rank}: invalid data-plane preflight envelope")
|
||||
continue
|
||||
error = item["error"]
|
||||
if error is not None:
|
||||
if not isinstance(error, str) or not error:
|
||||
errors.append(f"rank {rank}: invalid data-plane preflight error")
|
||||
else:
|
||||
errors.append(f"rank {rank}: {error}")
|
||||
continue
|
||||
mode = item["mode"]
|
||||
uid = item["uid_digest"]
|
||||
if mode not in {"tcp", "uid"} or (mode == "uid") != isinstance(uid, str):
|
||||
errors.append(f"rank {rank}: invalid data-plane identity")
|
||||
continue
|
||||
valid.append((rank, mode, uid))
|
||||
if not errors:
|
||||
_, mode, uid = valid[0]
|
||||
for rank, other_mode, other_uid in valid[1:]:
|
||||
if (other_mode, other_uid) != (mode, uid):
|
||||
errors.append(f"rank {rank}: NCCL data-plane identity disagrees")
|
||||
if errors:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n data-plane preflight rejected before NCCL init: "
|
||||
+ "; ".join(errors)
|
||||
)
|
||||
|
||||
|
||||
def validate_local_buffer(m2n: Any, meta: M2NParamMeta, buffer: Any) -> None:
|
||||
"""Check source dtype/shape while delegating buffer validity to M2N."""
|
||||
# A raw pointer carries no inspectable metadata. The source owns its
|
||||
# correctness; _send supplies the declared dtype and local shape to M2N.
|
||||
if isinstance(buffer, int):
|
||||
return
|
||||
|
||||
mesh, placements = resolve_layout(
|
||||
meta.source_layout.mesh, meta.source_layout.placements
|
||||
)
|
||||
actual = m2n.DistTensor(
|
||||
buffer,
|
||||
mesh=to_mesh(m2n, mesh),
|
||||
placements=to_placements(m2n, placements),
|
||||
)
|
||||
declared = m2n.DistTensor(
|
||||
None,
|
||||
local_shape=actual.local_shape,
|
||||
dtype=meta.dtype,
|
||||
mesh=to_mesh(m2n, mesh),
|
||||
placements=to_placements(m2n, placements),
|
||||
)
|
||||
if actual.dtype != declared.dtype:
|
||||
raise ValueError(
|
||||
f"parameter '{meta.name}' source returned dtype {actual.dtype}, "
|
||||
f"but its metadata declared {declared.dtype}"
|
||||
)
|
||||
|
||||
expected = expected_local_shape(meta)
|
||||
if actual.local_shape != expected:
|
||||
raise ValueError(
|
||||
f"parameter '{meta.name}' source returned local shape "
|
||||
f"{actual.local_shape}, but layout {meta.source_layout} implies "
|
||||
f"{expected} from global shape {meta.shape}"
|
||||
)
|
||||
|
||||
|
||||
def to_mesh(m2n: Any, mesh: M2NMesh) -> Any:
|
||||
return m2n.Mesh(mesh.dims, start_rank=mesh.start_rank)
|
||||
|
||||
|
||||
def to_placements(m2n: Any, placements: Placements) -> list[Any]:
|
||||
return [
|
||||
m2n.Replicate() if code == REPLICATE else m2n.Shard(code) for code in placements
|
||||
]
|
||||
|
||||
|
||||
def publish_destination_placements(
|
||||
comm: Any,
|
||||
device: torch.device,
|
||||
stream: torch.cuda.Stream,
|
||||
first_worker_rank: int,
|
||||
placements: "Sequence[Placements | None] | None",
|
||||
num_parameters: int,
|
||||
) -> list[Placements | None]:
|
||||
"""Share the worker-side destination plan with every rank in the group.
|
||||
|
||||
The trainer must issue each reshard with the same destination the workers
|
||||
use, but once destinations are per-parameter they depend on the inference
|
||||
model, which only the workers can see. The first worker publishes them here,
|
||||
over the shared NCCL communicator; trainer ranks pass `None` and receive
|
||||
them, and the other workers pass their own so a disagreement is caught
|
||||
rather than deadlocking later.
|
||||
|
||||
Only placements travel: the destination *mesh* is still derived from the
|
||||
rank split, identically on both sides. The plan is published on the data
|
||||
plane before any M2N reshard collective.
|
||||
"""
|
||||
replicated_sentinel = REPLICATE - 1
|
||||
encoded = torch.full(
|
||||
(num_parameters, MESH_NDIMS),
|
||||
replicated_sentinel,
|
||||
dtype=torch.int8,
|
||||
device=device,
|
||||
)
|
||||
if comm.rank == first_worker_rank:
|
||||
if placements is None or len(placements) != num_parameters:
|
||||
raise ValueError(
|
||||
f"publishing rank needs {num_parameters} destination placements"
|
||||
)
|
||||
rows = [
|
||||
[replicated_sentinel] * MESH_NDIMS if placement is REPLICATED else placement
|
||||
for placement in placements
|
||||
]
|
||||
if rows:
|
||||
encoded.copy_(torch.tensor(rows, dtype=torch.int8, device=device))
|
||||
|
||||
comm.broadcast(encoded, encoded, root=first_worker_rank, stream=stream)
|
||||
return [
|
||||
REPLICATED
|
||||
if all(code == replicated_sentinel for code in row)
|
||||
else cast(Placements, tuple(row))
|
||||
for row in encoded.tolist()
|
||||
]
|
||||
@@ -1,599 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-side weight transfer engine built on NCCL M2N (`nccl_m2n`).
|
||||
|
||||
The trainer and the inference workers share one NCCL communicator: trainer ranks
|
||||
occupy `[0, T)`, workers `[T, T + N)`. Each parameter is moved with a single
|
||||
`nccl.m2n.reshard`, which redistributes it from the trainer's layout (FSDP / EP
|
||||
/ arbitrary DTensor sharding) to the inference layout — so the trainer sends its
|
||||
local shards and never all-gathers a full tensor, which is what the broadcast
|
||||
NCCL backend forces it to do.
|
||||
|
||||
When an incoming checkpoint parameter maps directly to a live vLLM parameter,
|
||||
M2N reshards into that worker's local model storage. Parameters whose names or
|
||||
layouts cannot be resolved receive a full tensor and fall back to
|
||||
`load_weights`, preserving the existing backend's loading behavior.
|
||||
|
||||
Both meshes participate in every reshard, in the same order, so this backend has
|
||||
the same concurrency shape as the broadcast NCCL backend: the worker must be
|
||||
inside `receive_weights` while the trainer is sending. Driving that from the
|
||||
trainer is the trainer engine's job.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferEngine,
|
||||
WeightTransferInitInfo,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
DESTINATION_SHARD_AXIS,
|
||||
M2N_WIRE_SCHEMA_VERSION,
|
||||
MESH_NDIMS,
|
||||
M2NLayout,
|
||||
M2NMesh,
|
||||
M2NParamMeta,
|
||||
M2NWireParam,
|
||||
Placements,
|
||||
check_complete_plan_agreement,
|
||||
check_data_plane_agreement,
|
||||
check_runtime_ready_agreement,
|
||||
check_source_plan_agreement,
|
||||
check_transferable,
|
||||
create_m2n_communicator,
|
||||
expected_local_shape,
|
||||
import_m2n,
|
||||
prepare_m2n_local_runtime,
|
||||
publish_destination_placements,
|
||||
resolve_layout,
|
||||
source_plan_digest,
|
||||
to_mesh,
|
||||
to_placements,
|
||||
validate_transfer_plan,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_layout import (
|
||||
M2NDestination,
|
||||
resolve_parameter_destinations,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
decode_nccl_unique_id,
|
||||
worker_init_metadata_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"M2NWeightTransferInitInfo",
|
||||
"M2NWeightTransferUpdateInfo",
|
||||
"M2NWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wire types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class M2NWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Worker-side init info: the rendezvous plus the full transfer plan.
|
||||
|
||||
Layouts are static for the whole run, so they ride the one-time init
|
||||
handshake and the per-round update info stays a list of names. Everything is
|
||||
plain JSON so the HTTP control plane carries it unchanged.
|
||||
"""
|
||||
|
||||
schema_version: int
|
||||
master_address: str
|
||||
master_port: int
|
||||
rank_offset: int
|
||||
"""First worker rank, i.e. the number of trainer ranks."""
|
||||
world_size: int
|
||||
"""Trainer ranks + all inference workers."""
|
||||
dst_mesh_dims: list[int]
|
||||
"""The inference mesh, starting at `rank_offset`. Declared rather than
|
||||
derived so both sides describe the destination identically."""
|
||||
source_digest: str
|
||||
"""SHA-256 of the ordered, versioned source parameter plan."""
|
||||
params: list[dict[str, Any]]
|
||||
"""Ordered per-parameter source metadata and layouts."""
|
||||
nccl_unique_id_b64: str | None = field(default=None, repr=False)
|
||||
max_cta: int | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.master_address, str) or not self.master_address:
|
||||
raise ValueError("`master_address` must name the metadata rendezvous")
|
||||
if (
|
||||
not isinstance(self.master_port, int)
|
||||
or isinstance(self.master_port, bool)
|
||||
or not 0 < self.master_port < 65536
|
||||
):
|
||||
raise ValueError("`master_port` must be a valid metadata port")
|
||||
if (
|
||||
not isinstance(self.rank_offset, int)
|
||||
or isinstance(self.rank_offset, bool)
|
||||
or not isinstance(self.world_size, int)
|
||||
or isinstance(self.world_size, bool)
|
||||
or self.rank_offset < 1
|
||||
or self.rank_offset >= self.world_size
|
||||
):
|
||||
raise ValueError(
|
||||
f"`rank_offset` ({self.rank_offset}) must leave at least one "
|
||||
f"trainer rank and one worker in world_size {self.world_size}"
|
||||
)
|
||||
num_workers = self.world_size - self.rank_offset
|
||||
dst = tuple(self.dst_mesh_dims)
|
||||
try:
|
||||
dst_mesh = M2NMesh(cast(tuple[int, int], dst), self.rank_offset)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"invalid `dst_mesh_dims` {dst}: {exc}") from exc
|
||||
if dst_mesh.size != num_workers:
|
||||
raise ValueError(
|
||||
f"`dst_mesh_dims` {dst} must be {MESH_NDIMS} dims covering the "
|
||||
f"{num_workers} inference workers"
|
||||
)
|
||||
|
||||
def parse_wire_params(self) -> tuple[M2NWireParam, ...]:
|
||||
"""Parse and authenticate the ordered source plan."""
|
||||
if self.schema_version != M2N_WIRE_SCHEMA_VERSION:
|
||||
raise ValueError(
|
||||
"unsupported nccl_m2n init schema version "
|
||||
f"{self.schema_version}; expected {M2N_WIRE_SCHEMA_VERSION}"
|
||||
)
|
||||
if not isinstance(self.params, list):
|
||||
raise TypeError("`params` must be a list of wire parameter mappings")
|
||||
if not isinstance(self.source_digest, str) or not self.source_digest:
|
||||
raise ValueError("`source_digest` must be a non-empty string")
|
||||
wire_params: list[M2NWireParam] = []
|
||||
names: set[str] = set()
|
||||
for index, value in enumerate(self.params):
|
||||
if not isinstance(value, Mapping):
|
||||
raise TypeError(
|
||||
f"`params[{index}]` must be a mapping, got {type(value).__name__}"
|
||||
)
|
||||
param = M2NWireParam.from_dict(value)
|
||||
if param.name in names:
|
||||
raise ValueError(
|
||||
f"nccl_m2n init params contain duplicate parameter '{param.name}'"
|
||||
)
|
||||
names.add(param.name)
|
||||
wire_params.append(param)
|
||||
parsed = tuple(wire_params)
|
||||
computed_digest = source_plan_digest(parsed)
|
||||
if self.source_digest != computed_digest:
|
||||
raise ValueError(
|
||||
"nccl_m2n init source digest does not match params: "
|
||||
f"computed {computed_digest}, declared {self.source_digest}"
|
||||
)
|
||||
return parsed
|
||||
|
||||
@property
|
||||
def nccl_unique_id_bytes(self) -> bytes | None:
|
||||
if self.nccl_unique_id_b64 is None:
|
||||
return None
|
||||
return decode_nccl_unique_id(
|
||||
master_address=None,
|
||||
master_port=None,
|
||||
nccl_unique_id_b64=self.nccl_unique_id_b64,
|
||||
ctx="M2NWeightTransferInitInfo data plane",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class M2NWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Per-round update info: which parameters this chunk carries, in order.
|
||||
|
||||
Shapes, dtypes and layouts were fixed at init, so a round only needs to say
|
||||
what is coming and in what order — both sides must issue their reshards in
|
||||
exactly that order.
|
||||
"""
|
||||
|
||||
names: list[str]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Worker engine
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class M2NWeightTransferEngine(
|
||||
WeightTransferEngine[M2NWeightTransferInitInfo, M2NWeightTransferUpdateInfo]
|
||||
):
|
||||
"""Inference-side engine: receives each parameter with one reshard.
|
||||
|
||||
Resolvable parameters are resharded from the trainer layout directly into
|
||||
each worker's live local model storage. Unresolvable parameters receive a
|
||||
full tensor and use `load_weights`. In both cases, the trainer sends its
|
||||
local shards without materializing a full tensor.
|
||||
"""
|
||||
|
||||
init_info_cls = M2NWeightTransferInitInfo
|
||||
update_info_cls = M2NWeightTransferUpdateInfo
|
||||
# The destination plan is resolved against the model this engine was built
|
||||
# for, so it cannot be retargeted at a draft model mid-flight.
|
||||
supports_draft_weight_update = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: Any = None
|
||||
self._m2n: Any = None
|
||||
self._handle: Any = None
|
||||
self._metas: list[M2NParamMeta] = []
|
||||
self._dst_mesh: M2NMesh | None = None
|
||||
self._parameter_destinations: list[M2NDestination] = []
|
||||
self._index: dict[str, int] = {}
|
||||
self._uses_load_weights = False
|
||||
|
||||
def init_transfer_engine(self, init_info: M2NWeightTransferInitInfo) -> None:
|
||||
"""Agree on source and local readiness before constructing NCCL."""
|
||||
if self._handle is not None or self.model_update_group is not None:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n weight transfer is already initialized; call "
|
||||
"shutdown_weight_transfer_engine() before initializing again"
|
||||
)
|
||||
rendezvous_info = NCCLWeightTransferInitInfo(
|
||||
master_address=init_info.master_address,
|
||||
master_port=init_info.master_port,
|
||||
rank_offset=init_info.rank_offset,
|
||||
world_size=init_info.world_size,
|
||||
)
|
||||
metadata_group = worker_init_metadata_group(
|
||||
rendezvous_info, self.parallel_config
|
||||
)
|
||||
|
||||
wire_params: tuple[M2NWireParam, ...] = ()
|
||||
source_error: str | None = None
|
||||
try:
|
||||
self._m2n = import_m2n()
|
||||
wire_params = init_info.parse_wire_params()
|
||||
self._prepare_source_plan(wire_params, init_info.rank_offset)
|
||||
except Exception as exc:
|
||||
source_error = f"{type(exc).__name__}: {exc}"
|
||||
self._metas = []
|
||||
check_source_plan_agreement(
|
||||
metadata_group,
|
||||
wire_params,
|
||||
expected_digest=init_info.source_digest,
|
||||
local_error=source_error,
|
||||
)
|
||||
|
||||
destination_error: str | None = None
|
||||
try:
|
||||
self._prepare_destination_plan(init_info)
|
||||
except Exception as exc:
|
||||
destination_error = f"{type(exc).__name__}: {exc}"
|
||||
self._parameter_destinations = []
|
||||
|
||||
unique_id_bytes: bytes | None = None
|
||||
data_plane_error: str | None = None
|
||||
try:
|
||||
unique_id_bytes = init_info.nccl_unique_id_bytes
|
||||
except Exception as exc:
|
||||
data_plane_error = f"{type(exc).__name__}: {exc}"
|
||||
check_data_plane_agreement(
|
||||
metadata_group,
|
||||
unique_id_bytes,
|
||||
data_plane_error,
|
||||
)
|
||||
|
||||
handle = None
|
||||
group = None
|
||||
try:
|
||||
runtime_error = destination_error
|
||||
nccl = None
|
||||
device = None
|
||||
if runtime_error is None:
|
||||
try:
|
||||
nccl, device, handle = prepare_m2n_local_runtime(
|
||||
self._m2n, init_info.max_cta
|
||||
)
|
||||
except Exception as exc:
|
||||
runtime_error = f"{type(exc).__name__}: {exc}"
|
||||
check_runtime_ready_agreement(metadata_group, runtime_error)
|
||||
assert nccl is not None and device is not None and handle is not None
|
||||
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
|
||||
|
||||
# Keep the merged destination-placement publication unchanged: source
|
||||
# agreement is independent of worker-side destination selection.
|
||||
mine = [
|
||||
destination.placements for destination in self._parameter_destinations
|
||||
]
|
||||
agreed = publish_destination_placements(
|
||||
group,
|
||||
self.device,
|
||||
torch.cuda.current_stream(),
|
||||
init_info.rank_offset,
|
||||
mine,
|
||||
len(self._metas),
|
||||
)
|
||||
complete_plan_error: str | None = None
|
||||
try:
|
||||
if agreed != mine:
|
||||
mismatched = next(
|
||||
meta.name
|
||||
for meta, ours, theirs in zip(self._metas, mine, agreed)
|
||||
if ours != theirs
|
||||
)
|
||||
raise RuntimeError(
|
||||
"inference workers resolved different nccl_m2n "
|
||||
f"destinations (first mismatch: '{mismatched}'); all "
|
||||
"workers must run the same model and parallel config"
|
||||
)
|
||||
assert self._dst_mesh is not None
|
||||
validate_transfer_plan(self._metas, self._dst_mesh, agreed)
|
||||
except Exception as exc:
|
||||
complete_plan_error = f"{type(exc).__name__}: {exc}"
|
||||
assert self._dst_mesh is not None
|
||||
check_complete_plan_agreement(
|
||||
metadata_group,
|
||||
init_info.source_digest,
|
||||
self._dst_mesh,
|
||||
agreed,
|
||||
complete_plan_error,
|
||||
)
|
||||
except BaseException:
|
||||
if group is not None:
|
||||
try:
|
||||
# Transport cleanup only; reload validity belongs to the
|
||||
# model-update lifecycle, not this backend.
|
||||
group.abort()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to abort partially initialized nccl_m2n communicator"
|
||||
)
|
||||
if handle is not None:
|
||||
try:
|
||||
handle.destroy()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to destroy partially initialized nccl_m2n handle"
|
||||
)
|
||||
raise
|
||||
|
||||
self._handle = handle
|
||||
self.model_update_group = group
|
||||
|
||||
def _prepare_source_plan(
|
||||
self,
|
||||
wire_params: tuple[M2NWireParam, ...],
|
||||
num_trainer_ranks: int,
|
||||
) -> None:
|
||||
"""Validate worker-visible source metadata without constructing NCCL."""
|
||||
self._metas = []
|
||||
for param in wire_params:
|
||||
dtype = getattr(torch, param.dtype_name, None)
|
||||
if not isinstance(dtype, torch.dtype):
|
||||
raise ValueError(
|
||||
f"parameter '{param.name}' has invalid dtype name "
|
||||
f"'{param.dtype_name}'; expected the name of a torch.dtype"
|
||||
)
|
||||
check_transferable(param.name, dtype, param.shape)
|
||||
layout = M2NLayout(M2NMesh(param.src_mesh_dims, 0), param.src_placements)
|
||||
if layout.mesh.size != num_trainer_ranks:
|
||||
raise ValueError(
|
||||
f"parameter '{param.name}' source mesh covers "
|
||||
f"{layout.mesh.size} ranks, but there are "
|
||||
f"{num_trainer_ranks} trainer ranks"
|
||||
)
|
||||
meta = M2NParamMeta(param.name, dtype, param.shape, layout)
|
||||
expected_local_shape(meta)
|
||||
self._metas.append(meta)
|
||||
|
||||
def _prepare_destination_plan(self, init_info: M2NWeightTransferInitInfo) -> None:
|
||||
"""Resolve the merged-#51520 in-place/full-fallback destination plan."""
|
||||
# The inference topology, as declared by the trainer. This is the mesh
|
||||
# a sharded destination is placed over; a replicated one is described
|
||||
# by the alternate descriptor `resolve_layout` derives from it, which
|
||||
# covers the same ranks but is not this mesh.
|
||||
self._dst_mesh = M2NMesh(
|
||||
cast(tuple[int, int], tuple(init_info.dst_mesh_dims)),
|
||||
init_info.rank_offset,
|
||||
)
|
||||
parallel_config = self.parallel_config
|
||||
shard_axis_size = self._dst_mesh.dims[DESTINATION_SHARD_AXIS]
|
||||
quantization = getattr(self.model_config, "quantization", None)
|
||||
# A quantized weight loader may dequantize / transpose / requantize, and
|
||||
# under PP a rank's local shape is not the checkpoint shape split over
|
||||
# the shard axis. Both break shape-derived resolution, so take the
|
||||
# fallback.
|
||||
allow_direct = (
|
||||
quantization is None and parallel_config.pipeline_parallel_size == 1
|
||||
)
|
||||
# Match every incoming checkpoint parameter to its rank-local target.
|
||||
# Resolvable parameters can be resharded directly into live model
|
||||
# storage; the rest receive a full tensor for `load_weights`.
|
||||
self._parameter_destinations = resolve_parameter_destinations(
|
||||
self.model,
|
||||
[m.name for m in self._metas],
|
||||
[m.dtype for m in self._metas],
|
||||
[m.shape for m in self._metas],
|
||||
num_workers=self._dst_mesh.size,
|
||||
shard_axis_size=shard_axis_size,
|
||||
allow_direct=allow_direct,
|
||||
)
|
||||
validate_transfer_plan(
|
||||
self._metas,
|
||||
self._dst_mesh,
|
||||
[destination.placements for destination in self._parameter_destinations],
|
||||
)
|
||||
# Update requests carry names only, so cache their plan indices. Track
|
||||
# whether any fallback entry requires the `load_weights` lifecycle.
|
||||
self._index = {meta.name: i for i, meta in enumerate(self._metas)}
|
||||
self._uses_load_weights = any(
|
||||
not destination.direct for destination in self._parameter_destinations
|
||||
)
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Set up layerwise reloading, but only if some parameter needs it.
|
||||
|
||||
Directly-resharded parameters are written in place and never go through
|
||||
`load_weights`, so a plan with no fallback entries has nothing to
|
||||
reload.
|
||||
"""
|
||||
if not self._uses_load_weights:
|
||||
return
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
|
||||
initialize_layerwise_reload(self.model)
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Finalize layerwise reloading when the plan uses fallback entries."""
|
||||
if not self._uses_load_weights:
|
||||
return
|
||||
from vllm.model_executor.model_loader.reload import finalize_layerwise_reload
|
||||
|
||||
finalize_layerwise_reload(self.model, self.model_config)
|
||||
|
||||
def receive_weights(self, update_info: M2NWeightTransferUpdateInfo) -> None:
|
||||
"""Receive each requested parameter using its initialization-time plan."""
|
||||
if self._handle is None or self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"nccl_m2n weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
if not isinstance(update_info.names, list) or any(
|
||||
not isinstance(name, str) for name in update_info.names
|
||||
):
|
||||
raise TypeError("nccl_m2n update names must be a list of strings")
|
||||
|
||||
requested = []
|
||||
for name in update_info.names:
|
||||
index = self._index.get(name)
|
||||
if index is None:
|
||||
raise ValueError(
|
||||
f"parameter '{name}' was not declared at init; the "
|
||||
"trainer must send the same parameter set it announced"
|
||||
)
|
||||
requested.append((name, index))
|
||||
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
disable_mtp_completeness_check,
|
||||
)
|
||||
|
||||
stream = torch.cuda.current_stream()
|
||||
|
||||
# Reshard lazily so one `load_weights` invocation sees the complete
|
||||
# fallback sequence without staging every full tensor at once. Direct
|
||||
# destinations still execute in request order as the iterator advances.
|
||||
def reshard_requested_weights() -> Iterator[tuple[str, torch.Tensor]]:
|
||||
for name, index in requested:
|
||||
meta = self._metas[index]
|
||||
destination = self._parameter_destinations[index]
|
||||
buffer = destination.tensor
|
||||
if buffer is None:
|
||||
buffer = torch.empty(
|
||||
meta.shape, dtype=meta.dtype, device=self.device
|
||||
)
|
||||
|
||||
self._reshard(
|
||||
self.model_update_group,
|
||||
stream,
|
||||
meta,
|
||||
destination.placements,
|
||||
buffer,
|
||||
)
|
||||
|
||||
if destination.tensor is None:
|
||||
# `load_weights` reads on the host stream, so the transfer
|
||||
# has to have landed before it runs.
|
||||
stream.synchronize()
|
||||
yield name, buffer
|
||||
|
||||
has_fallback = any(
|
||||
self._parameter_destinations[index].tensor is None for _, index in requested
|
||||
)
|
||||
with disable_mtp_completeness_check():
|
||||
received_weights = reshard_requested_weights()
|
||||
if has_fallback:
|
||||
# Some model loaders keep invocation-local state to combine
|
||||
# related checkpoint tensors, so preserve one iterable scope.
|
||||
self.model.load_weights(received_weights)
|
||||
else:
|
||||
# The iterator performs the reshards. With nothing to yield to
|
||||
# `load_weights`, exhaust it here to execute direct transfers.
|
||||
for _ in received_weights:
|
||||
pass
|
||||
|
||||
def _reshard(
|
||||
self,
|
||||
comm: Any,
|
||||
stream: torch.cuda.Stream,
|
||||
src_meta: M2NParamMeta,
|
||||
dst_placements: Placements | None,
|
||||
dst_buffer: torch.Tensor,
|
||||
) -> None:
|
||||
"""Reshard one parameter from its trainer layout into a worker buffer.
|
||||
|
||||
`dst_placements` describes the buffer's logical placement over the
|
||||
worker mesh; `dst_buffer` is this rank's physical storage.
|
||||
"""
|
||||
assert self._dst_mesh is not None
|
||||
m2n = self._m2n
|
||||
src_mesh, resolved_src_placements = resolve_layout(
|
||||
src_meta.source_layout.mesh, src_meta.source_layout.placements
|
||||
)
|
||||
dst_mesh, resolved_dst_placements = resolve_layout(
|
||||
self._dst_mesh, dst_placements
|
||||
)
|
||||
m2n.reshard(
|
||||
None,
|
||||
dst_buffer,
|
||||
comm,
|
||||
stream,
|
||||
src_mesh=to_mesh(m2n, src_mesh),
|
||||
src_placements=to_placements(m2n, resolved_src_placements),
|
||||
dst_mesh=to_mesh(m2n, dst_mesh),
|
||||
dst_placements=to_placements(m2n, resolved_dst_placements),
|
||||
handle=self._handle,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Finish pending GPU work and release M2N communication state."""
|
||||
handle, self._handle = self._handle, None
|
||||
group, self.model_update_group = self.model_update_group, None
|
||||
self._m2n = None
|
||||
self._metas = []
|
||||
self._dst_mesh = None
|
||||
self._parameter_destinations = []
|
||||
self._index = {}
|
||||
self._uses_load_weights = False
|
||||
|
||||
first_error: BaseException | None = None
|
||||
|
||||
def attempt(action: Any, message: str) -> None:
|
||||
nonlocal first_error
|
||||
try:
|
||||
action()
|
||||
except BaseException as exc:
|
||||
if first_error is None:
|
||||
first_error = exc
|
||||
logger.exception(message)
|
||||
|
||||
if handle is not None or group is not None:
|
||||
# M2N does not synchronize caller streams on finalize.
|
||||
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
|
||||
if handle is not None:
|
||||
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
|
||||
if group is not None:
|
||||
attempt(group.abort, "failed to abort nccl_m2n communicator")
|
||||
if first_error is not None:
|
||||
raise first_error
|
||||
@@ -1,261 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Destination-layout resolution for the NCCL M2N weight transfer backend.
|
||||
|
||||
For every incoming checkpoint parameter the worker needs a destination buffer
|
||||
plus its placement over the inference mesh. Two outcomes are possible:
|
||||
|
||||
* **direct** — when the model is not quantized, the parameter maps 1:1 onto a
|
||||
live vLLM parameter and its loader is a small, known-safe copy or tensor-
|
||||
parallel loader. The loader's declared input/output dimension determines the
|
||||
placement; shapes are validation, not an inference mechanism. The reshard
|
||||
writes straight into the live parameter, so each rank receives only its own
|
||||
shard and nothing is copied afterwards.
|
||||
* **fallback** — anything else. The reshard delivers the whole tensor to every
|
||||
rank and `load_weights` does the sharding, exactly as the broadcast NCCL
|
||||
backend does. Fused parameters (`qkv_proj`, `gate_up_proj`, MoE `w13`/`w2`)
|
||||
take this path: the checkpoint name does not name a vLLM parameter, so there
|
||||
is nothing to resolve against.
|
||||
|
||||
Correctness never depends on a parameter resolving — the fallback is always
|
||||
available and is the same path the existing backend uses.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
DESTINATION_REPLICA_AXIS,
|
||||
DESTINATION_SHARD_AXIS,
|
||||
MESH_NDIMS,
|
||||
REPLICATE,
|
||||
REPLICATED,
|
||||
Placements,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_DIRECT_WEIGHT_LOADING_BLOCKLIST = {
|
||||
"vllm.model_executor.models.gpt2.GPT2Model",
|
||||
}
|
||||
|
||||
|
||||
def _module_type_name(module: torch.nn.Module) -> str:
|
||||
cls = type(module)
|
||||
return f"{cls.__module__}.{cls.__name__}"
|
||||
|
||||
|
||||
def _destination_placements(shard_dim: int) -> Placements:
|
||||
"""Place one tensor shard on the configured destination mesh axis."""
|
||||
placements = [REPLICATE] * MESH_NDIMS
|
||||
placements[DESTINATION_REPLICA_AXIS] = REPLICATE
|
||||
placements[DESTINATION_SHARD_AXIS] = shard_dim
|
||||
return cast(Placements, tuple(placements))
|
||||
|
||||
|
||||
@dataclass
|
||||
class M2NDestination:
|
||||
"""Where one checkpoint parameter lands on this worker."""
|
||||
|
||||
name: str
|
||||
placements: Placements | None
|
||||
"""Placement over the inference mesh, or `REPLICATED` for the fallback."""
|
||||
tensor: torch.Tensor | None
|
||||
"""The live parameter view to reshard into, or None for the fallback path
|
||||
(the engine allocates a full replica per round and calls `load_weights`)."""
|
||||
|
||||
@property
|
||||
def direct(self) -> bool:
|
||||
return self.tensor is not None
|
||||
|
||||
|
||||
def _base_loader_shard_dim(
|
||||
param: torch.nn.Parameter,
|
||||
shard_axis_size: int,
|
||||
) -> int | None:
|
||||
"""Return a known-safe loader's declared shard dim, or reject it.
|
||||
|
||||
Loader identity is deliberately fail-closed. A missing loader is not
|
||||
treated as ``default_weight_loader`` because a model-level ``load_weights``
|
||||
implementation may transform the tensor before it reaches that default.
|
||||
Wrappers and partials are also rejected: recognizing their wrapped callable
|
||||
would ignore behavior added by the wrapper.
|
||||
"""
|
||||
loader = getattr(param, "weight_loader", None)
|
||||
if loader is None or isinstance(loader, partial):
|
||||
return None
|
||||
|
||||
owner = loader.__self__ if inspect.ismethod(loader) else None
|
||||
loader_fn = loader.__func__ if inspect.ismethod(loader) else loader
|
||||
|
||||
# Lazy imports avoid making the weight-transfer registry import all model
|
||||
# executor layers during normal vLLM startup.
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
replicated_loaders = {
|
||||
default_weight_loader,
|
||||
ReplicatedLinear.weight_loader,
|
||||
}
|
||||
column_loaders = {
|
||||
ColumnParallelLinear.weight_loader,
|
||||
ColumnParallelLinear.weight_loader_v2,
|
||||
}
|
||||
row_loaders = {
|
||||
RowParallelLinear.weight_loader,
|
||||
RowParallelLinear.weight_loader_v2,
|
||||
}
|
||||
|
||||
# These attributes signal packing, a transform, or pre-sharded checkpoint
|
||||
# storage. Such semantics cannot be reproduced by a plain M2N placement.
|
||||
# Check them before accepting replicated loaders too: model-level loading
|
||||
# may transform a tensor before handing it to an otherwise plain copier.
|
||||
metadata_attrs = ("packed_dim", "packed_factor", "marlin_tile_size")
|
||||
flag_attrs = ("needs_scalar_to_array", "is_sharded_weight", "is_transposed")
|
||||
if any(getattr(param, attr, None) is not None for attr in metadata_attrs):
|
||||
return None
|
||||
if any(bool(getattr(param, attr, False)) for attr in flag_attrs):
|
||||
return None
|
||||
|
||||
if loader_fn in replicated_loaders:
|
||||
return REPLICATE
|
||||
if loader_fn not in column_loaders | row_loaders:
|
||||
return None
|
||||
|
||||
if owner is None or getattr(owner, "tp_size", None) != shard_axis_size:
|
||||
return None
|
||||
|
||||
attr = "output_dim" if loader_fn in column_loaders else "input_dim"
|
||||
dim = getattr(param, attr, None)
|
||||
return dim if isinstance(dim, int) and dim >= 0 else None
|
||||
|
||||
|
||||
def _validated_shard_dim(
|
||||
global_shape: Sequence[int],
|
||||
local_shape: Sequence[int],
|
||||
declared_dim: int,
|
||||
shard_axis_size: int,
|
||||
) -> int | None:
|
||||
"""Validate a loader-declared shard dimension against both shapes."""
|
||||
if len(global_shape) != len(local_shape):
|
||||
return None
|
||||
|
||||
if declared_dim == REPLICATE:
|
||||
return REPLICATE if tuple(global_shape) == tuple(local_shape) else None
|
||||
if declared_dim >= len(global_shape):
|
||||
return None
|
||||
for dim, (whole, local) in enumerate(zip(global_shape, local_shape)):
|
||||
expected = local * shard_axis_size if dim == declared_dim else local
|
||||
if whole != expected:
|
||||
return None
|
||||
return declared_dim
|
||||
|
||||
|
||||
def resolve_parameter_destinations(
|
||||
model: torch.nn.Module,
|
||||
names: Sequence[str],
|
||||
dtypes: Sequence[torch.dtype],
|
||||
shapes: Sequence[Sequence[int]],
|
||||
*,
|
||||
num_workers: int,
|
||||
shard_axis_size: int,
|
||||
allow_direct: bool,
|
||||
) -> list[M2NDestination]:
|
||||
"""Build this worker's destination plan, one entry per checkpoint parameter.
|
||||
|
||||
Placements are relative to the inference mesh: axis 0 replicates and axis 1
|
||||
shards. `allow_direct=False` forces every parameter onto the fallback path;
|
||||
the engine sets it for pipeline-parallel or quantized deployments, where a
|
||||
parameter's local shape is not simply the checkpoint shape split across the
|
||||
shard axis. Parameters below a model type with known model-level checkpoint
|
||||
preprocessing also fall back, since bypassing `load_weights` would skip the
|
||||
transformation.
|
||||
"""
|
||||
params = dict(model.named_parameters()) if allow_direct else {}
|
||||
blocked_param_ids = {
|
||||
id(param)
|
||||
for module in model.modules()
|
||||
if _module_type_name(module) in _DIRECT_WEIGHT_LOADING_BLOCKLIST
|
||||
for param in module.parameters()
|
||||
}
|
||||
|
||||
destinations: list[M2NDestination] = []
|
||||
for name, dtype, shape in zip(names, dtypes, shapes):
|
||||
param = params.get(name)
|
||||
dim = None
|
||||
if (
|
||||
param is not None
|
||||
and id(param) not in blocked_param_ids
|
||||
and param.dtype == dtype
|
||||
and param.data.is_contiguous()
|
||||
and num_workers % shard_axis_size == 0
|
||||
):
|
||||
declared_dim = _base_loader_shard_dim(param, shard_axis_size)
|
||||
if declared_dim is not None:
|
||||
dim = _validated_shard_dim(
|
||||
shape, param.shape, declared_dim, shard_axis_size
|
||||
)
|
||||
|
||||
if dim is None:
|
||||
# A replicated parameter is identical on every rank, so it needs no
|
||||
# placement of its own — REPLICATED lets resolve_layout spread it
|
||||
# over all the workers regardless of how the mesh is factored.
|
||||
destinations.append(M2NDestination(name, REPLICATED, None))
|
||||
elif dim == REPLICATE:
|
||||
assert param is not None
|
||||
destinations.append(M2NDestination(name, REPLICATED, param.data))
|
||||
else:
|
||||
assert param is not None
|
||||
destinations.append(
|
||||
M2NDestination(name, _destination_placements(dim), param.data)
|
||||
)
|
||||
|
||||
# Layerwise reload finalizes a module as a unit. Mixing direct and fallback
|
||||
# parameters within one module can copy stale storage over the direct load.
|
||||
fallback_modules = {
|
||||
name.rpartition(".")[0]
|
||||
for name, destination in zip(names, destinations)
|
||||
if not destination.direct and name in params
|
||||
}
|
||||
destinations = [
|
||||
M2NDestination(destination.name, REPLICATED, None)
|
||||
if destination.direct
|
||||
and destination.name.rpartition(".")[0] in fallback_modules
|
||||
else destination
|
||||
for destination in destinations
|
||||
]
|
||||
|
||||
num_direct = sum(d.direct for d in destinations)
|
||||
parameter_bytes = [
|
||||
math.prod(shape) * dtype.itemsize for dtype, shape in zip(dtypes, shapes)
|
||||
]
|
||||
total_bytes = sum(parameter_bytes)
|
||||
direct_bytes = sum(
|
||||
nbytes
|
||||
for destination, nbytes in zip(destinations, parameter_bytes)
|
||||
if destination.direct
|
||||
)
|
||||
direct_byte_percentage = 100 * direct_bytes / total_bytes if total_bytes else 0
|
||||
logger.info(
|
||||
"nccl_m2n destination plan: %d/%d parameters resharded directly into "
|
||||
"the model, %d via full-tensor fallback; direct byte coverage: "
|
||||
"%d/%d bytes (%.1f%%)",
|
||||
num_direct,
|
||||
len(destinations),
|
||||
len(destinations) - num_direct,
|
||||
direct_bytes,
|
||||
total_bytes,
|
||||
direct_byte_percentage,
|
||||
)
|
||||
return destinations
|
||||
@@ -1,207 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Trainer-side weight sources for the NCCL M2N backend.
|
||||
|
||||
m2n plans a transfer from both sides' layouts, so the trainer has to say how it
|
||||
holds each parameter — which the base `WeightSource` / `ParamMeta` pair does not
|
||||
express. This module supplies the m2n flavor of both, and the DTensor-backed
|
||||
implementation that covers the common trainer. Each parameter owns its source
|
||||
mesh and placements because dense and expert weights can use different
|
||||
factorizations over the same trainer ranks.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Iterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.base import WeightSource
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
REPLICATE,
|
||||
REPLICATED,
|
||||
M2NLayout,
|
||||
M2NMesh,
|
||||
M2NParamMeta,
|
||||
Placements,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"M2NWeightSource",
|
||||
"M2NManifestEntry",
|
||||
"ManifestM2NWeightSource",
|
||||
"DTensorModuleSource",
|
||||
"mesh_from_tensor",
|
||||
"placements_from_tensor",
|
||||
]
|
||||
|
||||
|
||||
class M2NWeightSource(WeightSource):
|
||||
"""A `WeightSource` that also describes how the trainer holds its weights.
|
||||
|
||||
Unlike `ModuleSource`, iteration yields each rank's **local shard**, not a
|
||||
materialized full tensor: gathering is exactly the cost m2n removes.
|
||||
"""
|
||||
|
||||
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
|
||||
"""Name, dtype, full shape, and source layout for each parameter."""
|
||||
raise NotImplementedError
|
||||
|
||||
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
|
||||
"""Yield `(name, local buffer)` pairs in metadata order.
|
||||
|
||||
Providers must enqueue materialization on the caller's current CUDA
|
||||
stream, or return a buffer whose writes are already visible to it.
|
||||
Buffers may be torch tensors, CUDA Array Interface objects, or raw
|
||||
device pointers accepted by nccl4py. Providers own correct
|
||||
communicator-device placement. Raw-pointer providers additionally own
|
||||
allocation lifetime, dtype, and shape correctness.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NManifestEntry:
|
||||
"""One immutable source descriptor with a lazy rank-local tensor view."""
|
||||
|
||||
key: str
|
||||
dtype: torch.dtype
|
||||
global_shape: tuple[int, ...]
|
||||
source_layout: M2NLayout
|
||||
local_tensor: Callable[[], Any]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "global_shape", tuple(self.global_shape))
|
||||
if not self.key:
|
||||
raise ValueError("M2N manifest key must not be empty")
|
||||
if not isinstance(self.dtype, torch.dtype):
|
||||
raise TypeError(
|
||||
f"manifest entry '{self.key}' has invalid dtype {self.dtype!r}"
|
||||
)
|
||||
if not isinstance(self.source_layout, M2NLayout):
|
||||
raise TypeError(
|
||||
f"manifest entry '{self.key}' source_layout must be an M2NLayout"
|
||||
)
|
||||
if not callable(self.local_tensor):
|
||||
raise TypeError(
|
||||
f"manifest entry '{self.key}' local_tensor must be callable"
|
||||
)
|
||||
|
||||
@property
|
||||
def metadata(self) -> M2NParamMeta:
|
||||
"""Return the serializable portion consumed during initialization."""
|
||||
return M2NParamMeta(
|
||||
self.key,
|
||||
self.dtype,
|
||||
self.global_shape,
|
||||
self.source_layout,
|
||||
)
|
||||
|
||||
|
||||
class ManifestM2NWeightSource(M2NWeightSource):
|
||||
"""An ordered M2N source backed by lazy, rank-local manifest entries."""
|
||||
|
||||
def __init__(self, entries: Sequence[M2NManifestEntry]) -> None:
|
||||
self._entries = tuple(entries)
|
||||
self._metadata = tuple(entry.metadata for entry in self._entries)
|
||||
|
||||
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
|
||||
"""Return a fresh list while preserving the immutable manifest order."""
|
||||
return list(self._metadata)
|
||||
|
||||
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
|
||||
"""Materialize one rank-local buffer at a time."""
|
||||
for entry in self._entries:
|
||||
yield entry.key, entry.local_tensor()
|
||||
|
||||
|
||||
def _placement_code(placement: Any) -> int:
|
||||
"""Map a `torch.distributed` placement onto an m2n placement code."""
|
||||
name = type(placement).__name__
|
||||
if name == "Replicate":
|
||||
return REPLICATE
|
||||
if name == "Shard":
|
||||
return int(placement.dim)
|
||||
raise ValueError(
|
||||
f"nccl_m2n cannot express the {name} placement; only Replicate and "
|
||||
"Shard are supported"
|
||||
)
|
||||
|
||||
|
||||
def mesh_from_tensor(tensor: torch.Tensor, num_trainer_ranks: int) -> M2NMesh:
|
||||
"""The mesh a parameter lives on, as an `M2NMesh`.
|
||||
|
||||
Tensors without DeviceMesh metadata are treated as replicated. Implicitly
|
||||
sharded tensors require a custom source with an explicit layout.
|
||||
"""
|
||||
device_mesh = getattr(tensor, "device_mesh", None)
|
||||
if device_mesh is None:
|
||||
return M2NMesh((num_trainer_ranks, 1), 0)
|
||||
|
||||
grid = device_mesh.mesh
|
||||
ranks = grid.flatten().tolist()
|
||||
if ranks != list(range(len(ranks))):
|
||||
raise ValueError(
|
||||
"nccl_m2n requires the trainer's device mesh to cover the "
|
||||
f"contiguous rank interval [0, {len(ranks)}); got {ranks}"
|
||||
)
|
||||
if grid.ndim > 2:
|
||||
raise ValueError(
|
||||
f"nccl_m2n supports 1-D and 2-D device meshes, got {grid.ndim}-D"
|
||||
)
|
||||
dims = tuple(grid.shape)
|
||||
return M2NMesh(dims if len(dims) == 2 else (dims[0], 1), 0)
|
||||
|
||||
|
||||
def placements_from_tensor(tensor: torch.Tensor) -> Placements | None:
|
||||
"""How a parameter is placed over its mesh, or `REPLICATED`.
|
||||
|
||||
`REPLICATED` is returned rather than a `(REPLICATE, REPLICATE)` pair: m2n
|
||||
cannot express that, and the size-1-axis encoding it needs instead is
|
||||
applied later by `resolve_layout`, which owns that workaround.
|
||||
"""
|
||||
placements = getattr(tensor, "placements", None)
|
||||
if placements is None:
|
||||
return REPLICATED
|
||||
codes = [_placement_code(p) for p in placements]
|
||||
if all(code == REPLICATE for code in codes):
|
||||
return REPLICATED
|
||||
if len(codes) == 1:
|
||||
return (codes[0], REPLICATE)
|
||||
return (codes[0], codes[1])
|
||||
|
||||
|
||||
class DTensorModuleSource(M2NWeightSource):
|
||||
"""`M2NWeightSource` over `module.named_parameters()`.
|
||||
|
||||
Covers both the FSDP/DTensor trainer (placement read off each parameter,
|
||||
local shard yielded via `to_local()`) and the plain replicated trainer, with
|
||||
no special casing. Trainers with a custom producer (a Megatron export, MoE
|
||||
re-fusing) subclass `M2NWeightSource` instead.
|
||||
"""
|
||||
|
||||
def __init__(self, module: torch.nn.Module, num_trainer_ranks: int) -> None:
|
||||
"""Wrap `module`; `num_trainer_ranks` sizes the mesh for plain tensors."""
|
||||
self._module = module
|
||||
self._num_trainer_ranks = num_trainer_ranks
|
||||
|
||||
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
|
||||
"""Read each global shape and source layout without gathering shards."""
|
||||
return [
|
||||
M2NParamMeta(
|
||||
name,
|
||||
p.dtype,
|
||||
tuple(p.shape),
|
||||
M2NLayout(
|
||||
mesh_from_tensor(p, self._num_trainer_ranks),
|
||||
placements_from_tensor(p),
|
||||
),
|
||||
)
|
||||
for name, p in self._module.named_parameters()
|
||||
]
|
||||
|
||||
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""Yield each parameter's local shard (`to_local()`), or the tensor itself."""
|
||||
for name, param in self._module.named_parameters():
|
||||
to_local = getattr(param, "to_local", None)
|
||||
yield name, (to_local() if callable(to_local) else param)
|
||||
@@ -1,628 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Trainer-side weight transfer engine for the NCCL M2N backend.
|
||||
|
||||
Symmetric to `M2NWeightTransferEngine` but in the training process. Every
|
||||
trainer rank joins the shared communicator and runs every reshard, sending its
|
||||
own local shard; only rank 0 touches the inference control plane.
|
||||
|
||||
Workers join the communicator during init and run reshard from inside
|
||||
`update_weights`, so those two RPCs overlap with work on this side. Both run on
|
||||
a helper thread and are joined afterwards, which keeps that overlap inside the
|
||||
engine -- where the `TrainerWeightTransferEngine` contract puts it -- rather than
|
||||
in every caller.
|
||||
"""
|
||||
|
||||
import socket
|
||||
from collections.abc import Mapping
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Self, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
TrainerInitInfo,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
M2N_WIRE_SCHEMA_VERSION,
|
||||
M2NMesh,
|
||||
M2NParamMeta,
|
||||
M2NWireParam,
|
||||
Placements,
|
||||
check_complete_plan_agreement,
|
||||
check_data_plane_agreement,
|
||||
check_runtime_ready_agreement,
|
||||
check_source_plan_agreement,
|
||||
check_transferable,
|
||||
create_m2n_communicator,
|
||||
expected_local_shape,
|
||||
import_m2n,
|
||||
prepare_m2n_local_runtime,
|
||||
publish_destination_placements,
|
||||
resolve_layout,
|
||||
source_plan_digest,
|
||||
to_mesh,
|
||||
to_placements,
|
||||
validate_local_buffer,
|
||||
validate_transfer_plan,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.m2n_source import M2NWeightSource
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
decode_nccl_unique_id,
|
||||
stateless_init_metadata_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = ["M2NTrainerInitInfo", "M2NTrainerWeightTransferEngine"]
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class M2NTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for nccl_m2n.
|
||||
|
||||
`rank` (from `TrainerInitInfo`) is this trainer process's rank; it is also
|
||||
its rank in the shared communicator, since the trainer occupies
|
||||
`[0, num_trainer_ranks)`. Rank 0 drives the control plane.
|
||||
"""
|
||||
|
||||
backend: ClassVar[str] = "nccl_m2n"
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
"""Trainer ranks + all inference workers."""
|
||||
num_trainer_ranks: int = 1
|
||||
dst_mesh_dims: tuple[int, int] | None = None
|
||||
"""How the inference ranks are laid out: axis 0 replicates and axis 1
|
||||
shards. The trainer declares it so both sides describe the destination
|
||||
identically; defaults to a flat `(num_workers, 1)`, which is all a
|
||||
replicated destination needs.
|
||||
|
||||
Destination meshes remain common to every parameter; per-parameter
|
||||
destination meshes are intentionally future work.
|
||||
"""
|
||||
nccl_unique_id_b64: str | None = field(default=None, repr=False)
|
||||
"""Optional UID for the NCCL data plane; TCP remains the metadata plane."""
|
||||
max_cta: int | None = None
|
||||
listen_socket: socket.socket | None = field(default=None, repr=False, compare=False)
|
||||
"""Optional rank-zero TCPStore reservation; never sent to workers."""
|
||||
|
||||
@property
|
||||
def destination_mesh_dims(self) -> tuple[int, int]:
|
||||
"""`dst_mesh_dims`, or a flat mesh over every inference worker."""
|
||||
if self.dst_mesh_dims is not None:
|
||||
return cast(tuple[int, int], tuple(self.dst_mesh_dims))
|
||||
return (self.world_size - self.num_trainer_ranks, 1)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Reject rank counts or destination meshes that cannot form a group."""
|
||||
if not isinstance(self.master_address, str) or not self.master_address:
|
||||
raise ValueError("`master_address` must name the metadata rendezvous")
|
||||
if (
|
||||
not isinstance(self.master_port, int)
|
||||
or isinstance(self.master_port, bool)
|
||||
or not 0 < self.master_port < 65536
|
||||
):
|
||||
raise ValueError("`master_port` must be a valid metadata port")
|
||||
if not 0 < self.num_trainer_ranks < self.world_size:
|
||||
raise ValueError(
|
||||
f"`num_trainer_ranks` ({self.num_trainer_ranks}) must leave at "
|
||||
f"least one worker in world_size {self.world_size}"
|
||||
)
|
||||
num_workers = self.world_size - self.num_trainer_ranks
|
||||
if self.dst_mesh_dims is not None:
|
||||
dims = tuple(self.dst_mesh_dims)
|
||||
try:
|
||||
mesh = M2NMesh(cast(tuple[int, int], dims), self.num_trainer_ranks)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"invalid `dst_mesh_dims` {dims}: {exc}") from exc
|
||||
if mesh.size != num_workers:
|
||||
raise ValueError(
|
||||
f"`dst_mesh_dims` {dims} must be two dims covering the "
|
||||
f"{num_workers} inference workers"
|
||||
)
|
||||
if not 0 <= self.rank < self.num_trainer_ranks:
|
||||
raise ValueError(
|
||||
f"trainer `rank` ({self.rank}) must be below "
|
||||
f"`num_trainer_ranks` ({self.num_trainer_ranks})"
|
||||
)
|
||||
if self.listen_socket is not None:
|
||||
if self.rank != 0:
|
||||
raise ValueError("only trainer rank zero may reserve the TCPStore")
|
||||
if not isinstance(self.listen_socket, socket.socket):
|
||||
raise TypeError("listen_socket must be a socket.socket")
|
||||
if self.listen_socket.fileno() < 0:
|
||||
raise ValueError("listen_socket is already closed")
|
||||
|
||||
@property
|
||||
def nccl_unique_id_bytes(self) -> bytes | None:
|
||||
if self.nccl_unique_id_b64 is None:
|
||||
return None
|
||||
return decode_nccl_unique_id(
|
||||
master_address=None,
|
||||
master_port=None,
|
||||
nccl_unique_id_b64=self.nccl_unique_id_b64,
|
||||
ctx="M2NTrainerInitInfo data plane",
|
||||
)
|
||||
|
||||
|
||||
class M2NTrainerWeightTransferEngine(TrainerWeightTransferEngine[M2NTrainerInitInfo]):
|
||||
"""Trainer-side engine: sends every rank's local shard, once per parameter.
|
||||
|
||||
Called on every trainer rank. All ranks join the communicator and run every
|
||||
reshard; only rank 0 touches the control plane.
|
||||
"""
|
||||
|
||||
init_info_cls = M2NTrainerInitInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: M2NWeightSource,
|
||||
is_sender: bool,
|
||||
) -> None:
|
||||
"""Hold the client and source; `trainer_init` fills in the group."""
|
||||
super().__init__(client=client, source=source, is_sender=is_sender)
|
||||
self.group: Any = None
|
||||
self._m2n: Any = None
|
||||
self._handle: Any = None
|
||||
self._metas: list[M2NParamMeta] = []
|
||||
self._wire_params: list[M2NWireParam] = []
|
||||
self._source_digest = ""
|
||||
self._dst_mesh: M2NMesh | None = None
|
||||
self._dst_placements: list[Placements | None] = []
|
||||
self._metadata_group: StatelessProcessGroup | None = None
|
||||
self._num_trainer_ranks = 0
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
init_info: M2NTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
"""Build the engine and rendezvous with the inference side.
|
||||
|
||||
Runs on *every* trainer rank. Rank 0 additionally ships the transfer
|
||||
plan to the workers and drives the control plane; the other ranks only
|
||||
build local state and join the communicator. Every rank participates
|
||||
in every reshard and sends its local shard.
|
||||
|
||||
Build a local validation envelope, start the worker RPC without
|
||||
waiting, join the metadata rendezvous, and require unanimous source,
|
||||
data-plane, and runtime readiness before constructing NCCL.
|
||||
"""
|
||||
engine = cls(
|
||||
client=client,
|
||||
source=cast(M2NWeightSource, source),
|
||||
is_sender=init_info.is_sender,
|
||||
)
|
||||
engine._initialize(init_info, source)
|
||||
return engine
|
||||
|
||||
def _initialize(
|
||||
self,
|
||||
init_info: M2NTrainerInitInfo,
|
||||
source: WeightSource | None,
|
||||
) -> None:
|
||||
"""Run source validation, metadata agreement, and NCCL rendezvous."""
|
||||
self._num_trainer_ranks = init_info.num_trainer_ranks
|
||||
local_source_error: str | None = None
|
||||
try:
|
||||
if not isinstance(source, M2NWeightSource):
|
||||
raise TypeError(
|
||||
"nccl_m2n needs per-parameter layouts, so its source must "
|
||||
f"be an M2NWeightSource; got {type(source).__name__}"
|
||||
)
|
||||
self._m2n = import_m2n()
|
||||
self._prepare_source_plan(source, init_info.num_trainer_ranks)
|
||||
except Exception as exc:
|
||||
local_source_error = f"{type(exc).__name__}: {exc}"
|
||||
self._metas = []
|
||||
self._wire_params = []
|
||||
self._source_digest = source_plan_digest([])
|
||||
|
||||
executor: ThreadPoolExecutor | None = None
|
||||
init_future: Future[None] | None = None
|
||||
if self.is_sender:
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="m2n-weight-sync"
|
||||
)
|
||||
init_future = executor.submit(
|
||||
self.client.init_weight_transfer_engine,
|
||||
self._worker_init_info(init_info),
|
||||
)
|
||||
handle = None
|
||||
group = None
|
||||
try:
|
||||
reservation = init_info.listen_socket
|
||||
try:
|
||||
metadata_group = stateless_init_metadata_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
init_info.rank,
|
||||
init_info.world_size,
|
||||
listen_socket=reservation,
|
||||
)
|
||||
finally:
|
||||
init_info.listen_socket = None
|
||||
if reservation is not None:
|
||||
reservation.close()
|
||||
plan_digest = check_source_plan_agreement(
|
||||
metadata_group,
|
||||
self._wire_params,
|
||||
expected_digest=self._source_digest,
|
||||
local_error=local_source_error,
|
||||
)
|
||||
|
||||
# Retain the common destination mesh and the merged NCCL publication
|
||||
# of per-parameter placements.
|
||||
self._dst_mesh = M2NMesh(
|
||||
init_info.destination_mesh_dims, init_info.num_trainer_ranks
|
||||
)
|
||||
unique_id_bytes: bytes | None = None
|
||||
data_plane_error: str | None = None
|
||||
try:
|
||||
unique_id_bytes = init_info.nccl_unique_id_bytes
|
||||
except Exception as exc:
|
||||
data_plane_error = f"{type(exc).__name__}: {exc}"
|
||||
check_data_plane_agreement(
|
||||
metadata_group,
|
||||
unique_id_bytes,
|
||||
data_plane_error,
|
||||
)
|
||||
|
||||
runtime_error: str | None = None
|
||||
nccl = None
|
||||
device = None
|
||||
try:
|
||||
nccl, device, handle = prepare_m2n_local_runtime(
|
||||
self._m2n, init_info.max_cta
|
||||
)
|
||||
except Exception as exc:
|
||||
runtime_error = f"{type(exc).__name__}: {exc}"
|
||||
check_runtime_ready_agreement(metadata_group, runtime_error)
|
||||
assert nccl is not None and device is not None and handle is not None
|
||||
group = create_m2n_communicator(nccl, metadata_group, unique_id_bytes)
|
||||
destination_placements = self._receive_destination_plan(
|
||||
group,
|
||||
init_info.num_trainer_ranks,
|
||||
torch.device("cuda", device),
|
||||
)
|
||||
|
||||
complete_plan_error: str | None = None
|
||||
try:
|
||||
assert self._dst_mesh is not None
|
||||
validate_transfer_plan(
|
||||
self._metas, self._dst_mesh, destination_placements
|
||||
)
|
||||
except Exception as exc:
|
||||
complete_plan_error = f"{type(exc).__name__}: {exc}"
|
||||
assert self._dst_mesh is not None
|
||||
check_complete_plan_agreement(
|
||||
metadata_group,
|
||||
plan_digest,
|
||||
self._dst_mesh,
|
||||
destination_placements,
|
||||
complete_plan_error,
|
||||
)
|
||||
|
||||
if init_future is not None:
|
||||
init_future.result()
|
||||
|
||||
logger.info(
|
||||
"nccl_m2n trainer ready: %d parameters, trainer ranks [0, %d), "
|
||||
"inference ranks [%d, %d)",
|
||||
len(self._metas),
|
||||
init_info.num_trainer_ranks,
|
||||
init_info.num_trainer_ranks,
|
||||
init_info.world_size,
|
||||
)
|
||||
logger.info("nccl_m2n source plan digest: %s", plan_digest)
|
||||
except BaseException:
|
||||
if group is not None:
|
||||
try:
|
||||
# Transport cleanup only; reload validity belongs to the
|
||||
# model-update lifecycle, not this backend.
|
||||
group.abort()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to abort partially initialized nccl_m2n communicator"
|
||||
)
|
||||
if handle is not None:
|
||||
try:
|
||||
handle.destroy()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to destroy partially initialized nccl_m2n handle"
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
if executor is not None:
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
self._metadata_group = metadata_group
|
||||
self._handle = handle
|
||||
self.group = group
|
||||
self._dst_placements = destination_placements
|
||||
|
||||
def _receive_destination_plan(
|
||||
self, group: Any, first_worker_rank: int, device: torch.device
|
||||
) -> list[Placements | None]:
|
||||
"""Receive the inference-side per-parameter placements."""
|
||||
# The first inference worker publishes one placement per parameter.
|
||||
# Every trainer rank receives the complete plan in this single,
|
||||
# initialization-time broadcast.
|
||||
placements = publish_destination_placements(
|
||||
group,
|
||||
device,
|
||||
torch.cuda.current_stream(),
|
||||
first_worker_rank,
|
||||
None,
|
||||
len(self._metas),
|
||||
)
|
||||
return placements
|
||||
|
||||
def _worker_init_info(self, init_info: M2NTrainerInitInfo) -> dict[str, Any]:
|
||||
"""Handshake payload: rendezvous, both meshes, and the transfer plan."""
|
||||
payload: dict[str, Any] = {
|
||||
"schema_version": M2N_WIRE_SCHEMA_VERSION,
|
||||
"master_address": init_info.master_address,
|
||||
"master_port": init_info.master_port,
|
||||
"nccl_unique_id_b64": init_info.nccl_unique_id_b64,
|
||||
"rank_offset": init_info.num_trainer_ranks,
|
||||
"world_size": init_info.world_size,
|
||||
"dst_mesh_dims": list(init_info.destination_mesh_dims),
|
||||
"source_digest": self._source_digest,
|
||||
"params": [param.to_dict() for param in self._wire_params],
|
||||
"max_cta": init_info.max_cta,
|
||||
}
|
||||
return {key: value for key, value in payload.items() if value is not None}
|
||||
|
||||
def _prepare_source_plan(
|
||||
self, source: M2NWeightSource, num_trainer_ranks: int
|
||||
) -> None:
|
||||
"""Build and validate this rank's immutable source plan."""
|
||||
self._metas = source.metadata()
|
||||
names: set[str] = set()
|
||||
for meta in self._metas:
|
||||
if not isinstance(meta, M2NParamMeta):
|
||||
raise TypeError(
|
||||
"nccl_m2n source metadata must contain M2NParamMeta "
|
||||
f"entries, got {type(meta).__name__}"
|
||||
)
|
||||
if meta.name in names:
|
||||
raise ValueError(
|
||||
"nccl_m2n source metadata contains duplicate parameter "
|
||||
f"'{meta.name}'"
|
||||
)
|
||||
names.add(meta.name)
|
||||
check_transferable(meta.name, meta.dtype, meta.shape)
|
||||
layout = meta.source_layout
|
||||
if layout.mesh.start_rank != 0:
|
||||
raise ValueError(
|
||||
f"parameter '{meta.name}' source mesh must start at rank 0, "
|
||||
f"got {layout.mesh.start_rank}"
|
||||
)
|
||||
if layout.mesh.size != num_trainer_ranks:
|
||||
raise ValueError(
|
||||
f"parameter '{meta.name}' source mesh covers "
|
||||
f"{layout.mesh.size} ranks, but there are "
|
||||
f"{num_trainer_ranks} trainer ranks"
|
||||
)
|
||||
expected_local_shape(meta)
|
||||
self._wire_params = [
|
||||
M2NWireParam(
|
||||
name=meta.name,
|
||||
dtype_name=str(meta.dtype).split(".")[-1],
|
||||
shape=meta.shape,
|
||||
src_mesh_dims=meta.source_layout.mesh.dims,
|
||||
src_placements=meta.source_layout.placements,
|
||||
)
|
||||
for meta in self._metas
|
||||
]
|
||||
self._source_digest = source_plan_digest(self._wire_params)
|
||||
|
||||
def send_weights(self) -> None:
|
||||
"""Drive one update round: start, reshard concurrently, then finish."""
|
||||
if self._handle is None or self.group is None:
|
||||
raise RuntimeError("nccl_m2n trainer engine is not initialized")
|
||||
|
||||
if not self.is_sender:
|
||||
try:
|
||||
self._send()
|
||||
except BaseException:
|
||||
try:
|
||||
self._receive_round_outcome()
|
||||
except BaseException:
|
||||
logger.exception(
|
||||
"failed to consume nccl_m2n round outcome after local failure"
|
||||
)
|
||||
raise
|
||||
self._receive_round_outcome()
|
||||
return
|
||||
|
||||
try:
|
||||
self.client.start_weight_update()
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="m2n-weight-sync"
|
||||
)
|
||||
try:
|
||||
update = executor.submit(
|
||||
self.client.update_weights,
|
||||
{"names": [m.name for m in self._metas]},
|
||||
)
|
||||
# Surface an immediate worker-side rejection before trainers
|
||||
# enter the collective sequence.
|
||||
if update.done():
|
||||
update.result()
|
||||
self._send()
|
||||
update.result()
|
||||
finally:
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
self.client.finish_weight_update()
|
||||
except BaseException as exc:
|
||||
try:
|
||||
self._publish_round_outcome(exc)
|
||||
except BaseException:
|
||||
logger.exception("failed to publish nccl_m2n round failure")
|
||||
raise
|
||||
|
||||
self._publish_round_outcome(None)
|
||||
|
||||
def _publish_round_outcome(self, failure: BaseException | None) -> None:
|
||||
"""Tell non-sender trainer ranks whether worker finalization passed."""
|
||||
if self._num_trainer_ranks <= 1:
|
||||
return
|
||||
if self._metadata_group is None:
|
||||
raise RuntimeError("nccl_m2n metadata group is unavailable")
|
||||
envelope = {
|
||||
"ok": failure is None,
|
||||
"error": (
|
||||
None if failure is None else f"{type(failure).__name__}: {failure}"
|
||||
),
|
||||
}
|
||||
for rank in range(1, self._num_trainer_ranks):
|
||||
self._metadata_group.send_obj(envelope, rank)
|
||||
# Keep rank zero's TCPStore alive until every trainer consumed its
|
||||
# outcome. This also keeps the per-rank message counters aligned.
|
||||
expected_ack = {"phase": "round_outcome_consumed"}
|
||||
for rank in range(1, self._num_trainer_ranks):
|
||||
ack = self._metadata_group.recv_obj(rank)
|
||||
if ack != expected_ack:
|
||||
raise RuntimeError(
|
||||
f"trainer rank {rank} sent an invalid round-outcome acknowledgement"
|
||||
)
|
||||
|
||||
def _receive_round_outcome(self) -> None:
|
||||
"""Wait until trainer rank zero observed every worker's finish RPC."""
|
||||
if self._metadata_group is None:
|
||||
raise RuntimeError("nccl_m2n metadata group is unavailable")
|
||||
outcome = self._metadata_group.recv_obj(0)
|
||||
self._metadata_group.send_obj({"phase": "round_outcome_consumed"}, 0)
|
||||
if not isinstance(outcome, Mapping) or set(outcome) != {"ok", "error"}:
|
||||
raise RuntimeError("trainer rank zero sent an invalid round outcome")
|
||||
ok = outcome["ok"]
|
||||
error = outcome["error"]
|
||||
if not isinstance(ok, bool) or ok != (error is None):
|
||||
raise RuntimeError("trainer rank zero sent an invalid round outcome")
|
||||
if not ok:
|
||||
if not isinstance(error, str) or not error:
|
||||
raise RuntimeError("trainer rank zero sent an invalid round error")
|
||||
raise RuntimeError(
|
||||
"nccl_m2n worker update failed on trainer rank zero: " + error
|
||||
)
|
||||
|
||||
def _send(self) -> None:
|
||||
"""Reshard each local shard into its worker-planned destination."""
|
||||
assert self.group is not None
|
||||
assert self._dst_mesh is not None
|
||||
assert self.source is not None
|
||||
m2n = self._m2n
|
||||
stream = torch.cuda.current_stream()
|
||||
|
||||
device_index = torch.accelerator.current_device_index()
|
||||
# Optimizer work may have used another CUDA stream. Make prior writes
|
||||
# visible before a provider materializes current-stream local shards.
|
||||
torch.accelerator.synchronize(device_index)
|
||||
inflight_buffers: list[Any] = []
|
||||
sent = 0
|
||||
for index, (name, buffer) in enumerate(self.source):
|
||||
if index >= len(self._metas):
|
||||
raise RuntimeError(
|
||||
"weight source yielded more parameters than metadata() "
|
||||
f"declared (first extra parameter: '{name}')"
|
||||
)
|
||||
meta = self._metas[index]
|
||||
if name != meta.name:
|
||||
raise RuntimeError(
|
||||
f"weight source yielded '{name}' at position {index}, but "
|
||||
f"its metadata declared '{meta.name}'; the source must "
|
||||
"iterate in a stable order"
|
||||
)
|
||||
validate_local_buffer(m2n, meta, buffer)
|
||||
src_mesh, src_placements = resolve_layout(
|
||||
meta.source_layout.mesh, meta.source_layout.placements
|
||||
)
|
||||
dst_mesh, dst_placements = resolve_layout(
|
||||
self._dst_mesh, self._dst_placements[index]
|
||||
)
|
||||
raw_pointer_args = {}
|
||||
if isinstance(buffer, int):
|
||||
raw_pointer_args = {
|
||||
"src_local_shape": expected_local_shape(meta),
|
||||
"src_dtype": meta.dtype,
|
||||
}
|
||||
m2n.reshard(
|
||||
buffer,
|
||||
None,
|
||||
self.group,
|
||||
stream,
|
||||
src_mesh=to_mesh(m2n, src_mesh),
|
||||
src_placements=to_placements(m2n, src_placements),
|
||||
dst_mesh=to_mesh(m2n, dst_mesh),
|
||||
dst_placements=to_placements(m2n, dst_placements),
|
||||
handle=self._handle,
|
||||
**raw_pointer_args,
|
||||
)
|
||||
# Let allocator-aware buffers recycle storage in stream order.
|
||||
# Other object owners must stay alive until the final synchronize;
|
||||
# raw-pointer lifetime remains the source's responsibility.
|
||||
record_stream = getattr(buffer, "record_stream", None)
|
||||
if callable(record_stream):
|
||||
record_stream(stream)
|
||||
elif not isinstance(buffer, int):
|
||||
inflight_buffers.append(buffer)
|
||||
sent += 1
|
||||
|
||||
if sent != len(self._metas):
|
||||
missing = self._metas[sent].name
|
||||
raise RuntimeError(
|
||||
f"weight source yielded {sent} parameters, but metadata() "
|
||||
f"declared {len(self._metas)} (first missing: '{missing}')"
|
||||
)
|
||||
|
||||
# As in the other weight-transfer backends, wait before returning
|
||||
# rather than relying on same-stream ordering. The caller may then
|
||||
# safely mutate parameters or continue work on another stream.
|
||||
stream.synchronize()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Finish pending GPU work and release M2N communication state."""
|
||||
handle, self._handle = self._handle, None
|
||||
group, self.group = self.group, None
|
||||
self._metadata_group = None
|
||||
self._dst_placements = []
|
||||
self._dst_mesh = None
|
||||
self._m2n = None
|
||||
|
||||
first_error: BaseException | None = None
|
||||
|
||||
def attempt(action: Any, message: str) -> None:
|
||||
nonlocal first_error
|
||||
try:
|
||||
action()
|
||||
except BaseException as exc:
|
||||
if first_error is None:
|
||||
first_error = exc
|
||||
logger.exception(message)
|
||||
|
||||
if handle is not None or group is not None:
|
||||
attempt(torch.accelerator.synchronize, "failed to synchronize nccl_m2n")
|
||||
if handle is not None:
|
||||
attempt(handle.destroy, "failed to destroy nccl_m2n handle")
|
||||
if group is not None:
|
||||
attempt(group.abort, "failed to abort nccl_m2n communicator")
|
||||
if first_error is not None:
|
||||
raise first_error
|
||||
@@ -1,41 +0,0 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Re-export the weight-transfer backend maintained by ModelExpress.
|
||||
|
||||
Install ModelExpress from the main branch's modelexpress_client/python
|
||||
subdirectory.
|
||||
"""
|
||||
|
||||
try:
|
||||
from modelexpress import configure_vllm_logging
|
||||
from modelexpress_rl.inference.engines.vllm.weight_transfer_engine import (
|
||||
ModelExpressWeightTransferEngine,
|
||||
ModelExpressWeightTransferInitInfo,
|
||||
ModelExpressWeightTransferUpdateInfo,
|
||||
)
|
||||
except ModuleNotFoundError as exc:
|
||||
if exc.name not in {
|
||||
"modelexpress",
|
||||
"modelexpress_rl",
|
||||
"modelexpress_rl.inference",
|
||||
"modelexpress_rl.inference.engines",
|
||||
"modelexpress_rl.inference.engines.vllm",
|
||||
"modelexpress_rl.inference.engines.vllm.weight_transfer_engine",
|
||||
}:
|
||||
raise
|
||||
raise ImportError(
|
||||
"The 'modelexpress' weight transfer backend requires ModelExpress. "
|
||||
"Install it with `uv pip install "
|
||||
"'git+https://github.com/ai-dynamo/modelexpress@main"
|
||||
"#subdirectory=modelexpress_client/python'`."
|
||||
) from exc
|
||||
|
||||
configure_vllm_logging()
|
||||
|
||||
__all__ = [
|
||||
"ModelExpressWeightTransferEngine",
|
||||
"ModelExpressWeightTransferInitInfo",
|
||||
"ModelExpressWeightTransferUpdateInfo",
|
||||
]
|
||||
@@ -1,299 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared NCCL initialization helpers for weight transfer engines.
|
||||
|
||||
The dense (`NCCLWeightTransferEngine`) and sparse
|
||||
(`SparseNCCLWeightTransferEngine`) backends are independent engines that share
|
||||
*only* their process-group initialization. That common logic lives here so the
|
||||
sparse engine does not have to subclass the dense one.
|
||||
"""
|
||||
|
||||
import socket
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
import pybase64 as base64
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||
NCCL_UNIQUE_ID_BYTES,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
DEFAULT_PACKED_NUM_BUFFERS,
|
||||
)
|
||||
from vllm.utils.nccl import unpinned_nccl_env
|
||||
|
||||
|
||||
def decode_nccl_unique_id(
|
||||
*,
|
||||
master_address: str | None,
|
||||
master_port: int | None,
|
||||
nccl_unique_id_b64: str | None,
|
||||
ctx: str,
|
||||
) -> bytes | None:
|
||||
"""Validate the rendezvous mode and decode a pre-shared unique id."""
|
||||
has_uid = bool(nccl_unique_id_b64)
|
||||
has_addr = master_address is not None
|
||||
has_port = master_port is not None
|
||||
|
||||
if has_uid and (has_addr or has_port):
|
||||
raise ValueError(
|
||||
f"{ctx}: pass nccl_unique_id_b64 OR master_address/master_port, not both"
|
||||
)
|
||||
if not has_uid:
|
||||
if not (has_addr and has_port):
|
||||
raise ValueError(
|
||||
f"{ctx}: need nccl_unique_id_b64, or both master_address and "
|
||||
f"master_port (got master_address={master_address!r}, "
|
||||
f"master_port={master_port!r})"
|
||||
)
|
||||
return None
|
||||
try:
|
||||
decoded = base64.b64decode(nccl_unique_id_b64, validate=True)
|
||||
except (ValueError, TypeError) as e:
|
||||
raise ValueError(
|
||||
f"{ctx}: nccl_unique_id_b64 is not valid standard base64 "
|
||||
f"(URL-safe base64 is not accepted)"
|
||||
) from e
|
||||
if len(decoded) != NCCL_UNIQUE_ID_BYTES:
|
||||
raise ValueError(
|
||||
f"{ctx}: nccl_unique_id_b64 must decode to {NCCL_UNIQUE_ID_BYTES} "
|
||||
f"bytes, got {len(decoded)}"
|
||||
)
|
||||
return decoded
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Worker-side initialization info for NCCL-based weight transfer backends.
|
||||
|
||||
Keyword-only (`kw_only`): adding the optional `nccl_unique_id_b64` field
|
||||
means the rendezvous fields can no longer keep a fixed positional slot, so a
|
||||
stale positional call fails loudly instead of silently swapping arguments.
|
||||
|
||||
Provide exactly one rendezvous mode:
|
||||
|
||||
* ``master_address`` + ``master_port`` -- TCPStore / ``StatelessProcessGroup``
|
||||
rendezvous (requires torch on every rank, including the trainer), or
|
||||
* ``nccl_unique_id_b64`` -- standard (RFC 4648, *not* URL-safe) base64 of the
|
||||
128 raw bytes from ``ncclGetUniqueId``, for torch-free trainers (e.g. JAX)
|
||||
that mint the unique id out of band and share it (over HTTP, etc.). Note a
|
||||
JAX peer must use ``base64.b64encode``, not ``urlsafe_b64encode``.
|
||||
|
||||
On the unique-id path all ranks must enter init concurrently (there is no
|
||||
store barrier), and every peer must honor the warm-up handshake: the
|
||||
worker's communicator issues a one-element ``all_reduce`` immediately after
|
||||
``ncclCommInitRank`` (see ``PyNcclCommunicator.from_unique_id_bytes``), so a
|
||||
foreign peer must issue a matching one-element ``all_reduce`` before any
|
||||
other collective or all ranks deadlock.
|
||||
"""
|
||||
|
||||
master_address: str | None = None
|
||||
master_port: int | None = None
|
||||
rank_offset: int
|
||||
world_size: int
|
||||
nccl_unique_id_b64: str | None = field(default=None, repr=False)
|
||||
packed: bool = False
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_ = self.nccl_unique_id_bytes
|
||||
|
||||
@property
|
||||
def nccl_unique_id_bytes(self) -> bytes | None:
|
||||
return decode_nccl_unique_id(
|
||||
master_address=self.master_address,
|
||||
master_port=self.master_port,
|
||||
nccl_unique_id_b64=self.nccl_unique_id_b64,
|
||||
ctx="NCCLWeightTransferInitInfo",
|
||||
)
|
||||
|
||||
|
||||
def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
|
||||
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
|
||||
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
|
||||
only the mode actually in use. Shared by the dense and sparse trainer
|
||||
engines so the two cannot drift."""
|
||||
return {key: value for key, value in asdict(init_info).items() if value is not None}
|
||||
|
||||
|
||||
class NCCLRendezvous(Protocol):
|
||||
"""The TCP rendezvous fields `trainer_init` needs.
|
||||
|
||||
Structural so each backend can keep its own trainer init info next to its
|
||||
engine (dense `NCCLTrainerInitInfo`, `SparseNCCLTrainerInitInfo`) without
|
||||
this shared module importing either.
|
||||
"""
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
|
||||
|
||||
def stateless_init_metadata_group(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
*,
|
||||
listen_socket: socket.socket | None = None,
|
||||
) -> "StatelessProcessGroup":
|
||||
"""Create only the TCPStore group used for metadata agreement."""
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
return StatelessProcessGroup.create(
|
||||
host=master_address,
|
||||
port=master_port,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
listen_socket=listen_socket,
|
||||
)
|
||||
|
||||
|
||||
def stateless_init_process_group(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
device,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""VLLM provides `StatelessProcessGroup` to create a process group
|
||||
without considering the global process group in torch.distributed.
|
||||
It is recommended to create `StatelessProcessGroup`, and then initialize
|
||||
the data-plane communication (NCCL) between external (train processes)
|
||||
and vLLM workers.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
pg = StatelessProcessGroup.create(
|
||||
host=master_address, port=master_port, rank=rank, world_size=world_size
|
||||
)
|
||||
with unpinned_nccl_env():
|
||||
return PyNcclCommunicator(pg, device=device)
|
||||
|
||||
|
||||
def uid_init_process_group(
|
||||
nccl_unique_id_bytes: bytes,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
device,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Join the NCCL group from pre-shared ``ncclUniqueId`` bytes.
|
||||
|
||||
The torch-free rendezvous alternative to `stateless_init_process_group`: no
|
||||
TCPStore, and therefore no barrier -- every rank must enter concurrently.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
with unpinned_nccl_env():
|
||||
return PyNcclCommunicator.from_unique_id_bytes(
|
||||
nccl_unique_id_bytes,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
def _worker_group_rank(rank_offset: int, parallel_config: "ParallelConfig") -> int:
|
||||
"""Return this worker's rank in a trainer-to-worker process group."""
|
||||
dp_rank = parallel_config.data_parallel_index
|
||||
world_size_per_dp = parallel_config.world_size # TP * PP
|
||||
rank_within_dp = parallel_config.rank
|
||||
return rank_offset + dp_rank * world_size_per_dp + rank_within_dp
|
||||
|
||||
|
||||
def worker_init_metadata_group(
|
||||
init_info: NCCLWeightTransferInitInfo,
|
||||
parallel_config: "ParallelConfig",
|
||||
) -> "StatelessProcessGroup":
|
||||
"""Join the trainer-to-worker TCP metadata group without constructing NCCL."""
|
||||
assert init_info.master_address is not None
|
||||
assert init_info.master_port is not None
|
||||
return stateless_init_metadata_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
_worker_group_rank(init_info.rank_offset, parallel_config),
|
||||
init_info.world_size,
|
||||
)
|
||||
|
||||
|
||||
def worker_init_process_group(
|
||||
init_info: NCCLWeightTransferInitInfo,
|
||||
parallel_config: "ParallelConfig",
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Create the trainer<->worker NCCL group on an inference worker.
|
||||
|
||||
Computes a unique rank for this worker across all data-parallel groups and
|
||||
joins the trainer via whichever rendezvous mode `init_info` carries.
|
||||
"""
|
||||
rank = _worker_group_rank(init_info.rank_offset, parallel_config)
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
unique_id_bytes = init_info.nccl_unique_id_bytes
|
||||
if unique_id_bytes is not None:
|
||||
return uid_init_process_group(
|
||||
unique_id_bytes,
|
||||
rank,
|
||||
init_info.world_size,
|
||||
device,
|
||||
)
|
||||
# __post_init__ guarantees the TCP pair when there is no unique id.
|
||||
assert init_info.master_address is not None
|
||||
assert init_info.master_port is not None
|
||||
return stateless_init_process_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
rank,
|
||||
init_info.world_size,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
def trainer_init(
|
||||
init_info: NCCLRendezvous | dict,
|
||||
rank: int = 0,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Initialize NCCL process group for trainer-side weight transfer.
|
||||
|
||||
Uses the current CUDA device (torch.accelerator.current_device_index()).
|
||||
|
||||
Args:
|
||||
init_info: Any object carrying the `NCCLRendezvous` fields (a trainer or
|
||||
worker NCCL init info), or a dict with keys:
|
||||
- master_address: str
|
||||
- master_port: int
|
||||
- world_size: int
|
||||
rank: This trainer process's rank in the group. The broadcast backends
|
||||
have a single trainer at rank 0; multi-rank trainers (m2n) occupy
|
||||
ranks `[0, num_trainer_ranks)` and the workers start after them.
|
||||
|
||||
Returns:
|
||||
PyNcclCommunicator for weight transfer.
|
||||
|
||||
"""
|
||||
if isinstance(init_info, dict):
|
||||
master_address = init_info["master_address"]
|
||||
master_port = init_info["master_port"]
|
||||
world_size = init_info["world_size"]
|
||||
else:
|
||||
master_address = init_info.master_address
|
||||
master_port = init_info.master_port
|
||||
world_size = init_info.world_size
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
return stateless_init_process_group(
|
||||
master_address,
|
||||
master_port,
|
||||
rank,
|
||||
world_size,
|
||||
device,
|
||||
)
|
||||
@@ -1,461 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NCCL-based (dense) weight transfer engine."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar, Self
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
ParamMeta,
|
||||
TrainerInitInfo,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
WeightTransferEngine,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
worker_init_payload,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
trainer_init as open_trainer_endpoint,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
DEFAULT_PACKED_NUM_BUFFERS,
|
||||
packed_nccl_broadcast_consumer,
|
||||
packed_nccl_broadcast_producer,
|
||||
)
|
||||
|
||||
# NCCLWeightTransferInitInfo is re-exported here for convenience; its canonical
|
||||
# home is nccl_common, shared with the sparse backend.
|
||||
__all__ = [
|
||||
"NCCLWeightTransferInitInfo",
|
||||
"NCCLTrainerInitInfo",
|
||||
"NCCLWeightTransferUpdateInfo",
|
||||
"NCCLWeightTransferEngine",
|
||||
"NCCLTrainerWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for the dense NCCL weight transfer backend.
|
||||
|
||||
The sender opens its endpoint as NCCL rank 0, so it needs no `rank_offset`.
|
||||
`world_size` is the full trainer+worker NCCL group size. `rank` (from
|
||||
`TrainerInitInfo`) identifies this trainer process; rank 0 is the sender.
|
||||
|
||||
The trainer joins over a TCPStore rendezvous (`master_address` +
|
||||
`master_port`). Torch-free trainers (e.g. JAX) that cannot join a TCPStore
|
||||
mint an `ncclUniqueId` themselves and drive rank 0 out of band; they ship a
|
||||
`nccl_unique_id_b64` payload straight to the inference workers'
|
||||
`init_weight_transfer_engine` (see `NCCLWeightTransferInitInfo`) rather than
|
||||
going through this engine.
|
||||
|
||||
`packed` / buffer sizes are the transfer's wire params. The trainer
|
||||
propagates them to the worker at `trainer_init` so the two sides cannot
|
||||
disagree. Note this defaults to packed, unlike the worker-side
|
||||
`NCCLWeightTransferInitInfo`, whose default only applies when no trainer
|
||||
ships a value. `backend` is the factory dispatch key."""
|
||||
|
||||
backend: ClassVar[str] = "nccl"
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
packed: bool = True
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Per-round update info for the dense NCCL weight transfer backend.
|
||||
|
||||
Whether the transfer is packed (and the buffer geometry) is a must-agree
|
||||
wire param carried on the init info (`NCCLTrainerInitInfo` /
|
||||
`NCCLWeightTransferInitInfo`), not here; this carries only the per-round
|
||||
parameter metadata.
|
||||
"""
|
||||
|
||||
names: list[str]
|
||||
dtype_names: list[str]
|
||||
shapes: list[list[int]]
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate that all lists have the same length."""
|
||||
num_params = len(self.names)
|
||||
if len(self.dtype_names) != num_params:
|
||||
raise ValueError(
|
||||
f"`dtype_names` should be of the same size as `names`: "
|
||||
f"got {len(self.dtype_names)} and {len(self.names)}"
|
||||
)
|
||||
if len(self.shapes) != num_params:
|
||||
raise ValueError(
|
||||
f"`shapes` should be of the same size as `names`: "
|
||||
f"got {len(self.shapes)} and {len(self.names)}"
|
||||
)
|
||||
|
||||
|
||||
class NCCLWeightTransferEngine(
|
||||
WeightTransferEngine[NCCLWeightTransferInitInfo, NCCLWeightTransferUpdateInfo]
|
||||
):
|
||||
"""Weight transfer engine using NCCL for communication between trainer and workers.
|
||||
|
||||
This implementation uses NCCL broadcast operations to transfer dense
|
||||
checkpoint-format weights from the trainer (rank 0) to all inference workers
|
||||
in a process group. Received weights are loaded via the model's
|
||||
`load_weights` using the layerwise reload lifecycle.
|
||||
"""
|
||||
|
||||
# Define backend-specific dataclass types
|
||||
init_info_cls = NCCLWeightTransferInitInfo
|
||||
update_info_cls = NCCLWeightTransferUpdateInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
# Set from the trainer-supplied init info at the handshake; defaults are
|
||||
# only for the (unreachable) receive-before-init case.
|
||||
self.packed = False
|
||||
self.packed_buffer_size_bytes = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
self.packed_num_buffers = DEFAULT_PACKED_NUM_BUFFERS
|
||||
|
||||
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
|
||||
"""Initialize NCCL process group with the trainer and record the
|
||||
trainer-supplied wire params so the worker decodes exactly as the
|
||||
trainer encodes.
|
||||
|
||||
Args:
|
||||
init_info: NCCL initialization info containing master address, port,
|
||||
rank offset, world size, and the packed wire params
|
||||
|
||||
"""
|
||||
self.packed = init_info.packed
|
||||
self.packed_buffer_size_bytes = init_info.packed_buffer_size_bytes
|
||||
self.packed_num_buffers = init_info.packed_num_buffers
|
||||
self.model_update_group = worker_init_process_group(
|
||||
init_info, self.parallel_config
|
||||
)
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Initialize layerwise reloading for the incoming checkpoint weights."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
|
||||
initialize_layerwise_reload(self.model)
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Finalize layerwise reloading after all weights have been received."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
)
|
||||
|
||||
finalize_layerwise_reload(self.model, self.model_config)
|
||||
|
||||
def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None:
|
||||
"""Receive weights from trainer via NCCL broadcast.
|
||||
|
||||
Whether to use packed broadcasting (and the buffer geometry) is read
|
||||
from `self.packed` / `self.packed_*`, set at the init handshake from the
|
||||
trainer's init info, so it is guaranteed to match how the trainer
|
||||
encoded.
|
||||
|
||||
Args:
|
||||
update_info: NCCL update info containing parameter names, dtypes,
|
||||
and shapes
|
||||
|
||||
"""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"NCCL weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
disable_mtp_completeness_check,
|
||||
)
|
||||
|
||||
with disable_mtp_completeness_check():
|
||||
if self.packed:
|
||||
# Build iterator of (name, (shape, dtype)) from update_info
|
||||
def state_dict_info_iterator():
|
||||
for name, dtype_name, shape in zip(
|
||||
update_info.names, update_info.dtype_names, update_info.shapes
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
yield (name, (shape, dtype))
|
||||
|
||||
packed_nccl_broadcast_consumer(
|
||||
iterator=state_dict_info_iterator(),
|
||||
group=self.model_update_group,
|
||||
src=0,
|
||||
post_unpack_func=self.model.load_weights,
|
||||
buffer_size_bytes=self.packed_buffer_size_bytes,
|
||||
num_buffers=self.packed_num_buffers,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
# Use simple one-by-one broadcasting
|
||||
for name, dtype_name, shape in zip(
|
||||
update_info.names, update_info.dtype_names, update_info.shapes
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
weight = torch.empty(shape, dtype=dtype, device=self.device)
|
||||
self.model_update_group.broadcast(
|
||||
weight, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
self.model.load_weights([(name, weight)])
|
||||
del weight
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
|
||||
|
||||
class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerInitInfo]):
|
||||
"""Trainer-side NCCL weight transfer engine.
|
||||
|
||||
On the sender (rank 0) holds the NCCL communicator and drives the full
|
||||
update round trip: it runs the inference-side `update_weights` concurrently
|
||||
with the trainer-side broadcast (both rendezvous inside the same NCCL
|
||||
calls), then finishes the update. Non-sender trainer ranks hold no
|
||||
communicator; they only iterate the source to stay in the trainer-side
|
||||
collective (e.g. FSDP `full_tensor()`) and skip the client RPCs and the
|
||||
broadcast (all guarded on `is_sender`).
|
||||
|
||||
`packed` / buffer sizes come from `NCCLTrainerInitInfo`; the sender
|
||||
propagates them to the worker at `trainer_init` (on the worker-side init
|
||||
info), so per-round payloads carry only parameter metadata.
|
||||
"""
|
||||
|
||||
init_info_cls = NCCLTrainerInitInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource,
|
||||
is_sender: bool = True,
|
||||
packed: bool = True,
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
|
||||
) -> None:
|
||||
super().__init__(client=client, source=source, is_sender=is_sender)
|
||||
self.packed = packed
|
||||
self.packed_buffer_size_bytes = packed_buffer_size_bytes
|
||||
self.packed_num_buffers = packed_num_buffers
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
init_info: NCCLTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
if source is None:
|
||||
raise ValueError("NCCL trainer weight transfer requires a WeightSource.")
|
||||
engine = cls(
|
||||
client=client,
|
||||
source=source,
|
||||
is_sender=init_info.is_sender,
|
||||
packed=init_info.packed,
|
||||
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
|
||||
packed_num_buffers=init_info.packed_num_buffers,
|
||||
)
|
||||
if not engine.is_sender:
|
||||
# Non-sender trainer ranks aren't part of the transfer NCCL group and
|
||||
# don't drive the inference side; they only participate in the
|
||||
# trainer-side gather during send_weights.
|
||||
return engine
|
||||
|
||||
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
|
||||
worker_init_info = NCCLWeightTransferInitInfo(
|
||||
master_address=init_info.master_address,
|
||||
master_port=init_info.master_port,
|
||||
rank_offset=1,
|
||||
world_size=init_info.world_size,
|
||||
packed=init_info.packed,
|
||||
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
|
||||
packed_num_buffers=init_info.packed_num_buffers,
|
||||
)
|
||||
|
||||
# The inference workers block inside init_weight_transfer_engine waiting
|
||||
# for the NCCL rendezvous, so we kick that off on a side thread while we
|
||||
# open the trainer endpoint (rank 0); both sides must rendezvous together.
|
||||
with ThreadPoolExecutor(max_workers=1) as exe:
|
||||
future = exe.submit(
|
||||
engine.client.init_weight_transfer_engine,
|
||||
worker_init_payload(worker_init_info),
|
||||
)
|
||||
engine.model_update_group = open_trainer_endpoint(init_info)
|
||||
future.result() # surface any inference-side init error
|
||||
|
||||
return engine
|
||||
|
||||
def send_weights(self) -> None:
|
||||
assert self.source is not None # guaranteed by trainer_init / __init__
|
||||
source = self.source
|
||||
|
||||
# Metadata is declared without gathering. For Megatron it is itself a
|
||||
# collective, so every rank runs it; only the sender ships it.
|
||||
meta = source.metadata()
|
||||
|
||||
if not self.is_sender:
|
||||
# Non-sender ranks only join the trainer-side gather collective.
|
||||
self._broadcast(source, meta)
|
||||
self._post_send_sync()
|
||||
return
|
||||
|
||||
update_info = NCCLWeightTransferUpdateInfo(
|
||||
names=[m.name for m in meta],
|
||||
dtype_names=[str(m.dtype).split(".")[-1] for m in meta],
|
||||
shapes=[list(m.shape) for m in meta],
|
||||
)
|
||||
|
||||
self.client.start_weight_update()
|
||||
# update_weights (workers receive) must run concurrently with the
|
||||
# trainer-side broadcast — both rendezvous inside the same NCCL calls.
|
||||
exe = ThreadPoolExecutor(max_workers=1)
|
||||
try:
|
||||
future = exe.submit(self.client.update_weights, asdict(update_info))
|
||||
# Cheap best-effort: if update_weights already failed (e.g. a bad
|
||||
# request rejected before any NCCL call), surface it now instead of
|
||||
# hanging in broadcast waiting for a peer that will never arrive.
|
||||
if future.done():
|
||||
future.result()
|
||||
self._broadcast(source, meta)
|
||||
future.result() # surface inference-side errors
|
||||
finally:
|
||||
# Never wait for the RPC thread here. If the broadcast raised, the
|
||||
# worker is still blocked in the matching NCCL call and will never
|
||||
# return, so joining would turn the error into a permanent hang.
|
||||
# Let the exception out instead; the transfer group is unusable
|
||||
# either way and the caller has to tear it down. (The orphaned
|
||||
# thread is still joined at interpreter exit, so a caller that wants
|
||||
# to exit cleanly after such a failure must not wait on it.)
|
||||
exe.shutdown(wait=False)
|
||||
self.client.finish_weight_update()
|
||||
self._post_send_sync()
|
||||
|
||||
def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None:
|
||||
"""Iterate the source (materializing each tensor — a collective on all
|
||||
ranks) and, on the sender, broadcast from rank 0, packed or one-by-one.
|
||||
Non-sender ranks only replay the iteration to stay in the collective."""
|
||||
if not self.is_sender:
|
||||
for _ in source:
|
||||
pass
|
||||
return
|
||||
|
||||
assert self.model_update_group is not None, (
|
||||
"trainer_init() must be called before _broadcast()."
|
||||
)
|
||||
pairs = self._checked_iter(source, meta)
|
||||
if self.packed:
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=pairs,
|
||||
group=self.model_update_group,
|
||||
src=0,
|
||||
post_iter_func=lambda item: item[1],
|
||||
buffer_size_bytes=self.packed_buffer_size_bytes,
|
||||
num_buffers=self.packed_num_buffers,
|
||||
)
|
||||
else:
|
||||
stream = torch.cuda.current_stream()
|
||||
for _name, tensor in pairs:
|
||||
# NCCL sends `numel` elements straight from `data_ptr()`, so a
|
||||
# non-contiguous view would ship whatever follows its base
|
||||
# pointer. Keep the copy referenced until the broadcast is
|
||||
# enqueued. (The packed path linearizes in `pack_tensors`.)
|
||||
send = tensor if tensor.is_contiguous() else tensor.contiguous()
|
||||
self.model_update_group.broadcast(send, src=0, stream=stream)
|
||||
|
||||
@staticmethod
|
||||
def _checked_iter(
|
||||
source: WeightSource, meta: list[ParamMeta]
|
||||
) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""Yield the source's pairs, checking each against what the worker was
|
||||
told to expect.
|
||||
|
||||
The worker sizes its receive buffers — and in packed mode cuts its chunk
|
||||
boundaries — from the update info, which is built from `metadata()`. If
|
||||
iteration disagrees with it, the two sides split the stream differently
|
||||
and the transfer hangs in NCCL or loads garbage. Checking here costs one
|
||||
comparison per parameter and turns that into an error naming the first
|
||||
divergent parameter. Sender-only: under pipeline parallelism a
|
||||
non-sender's yielded tensor is not meaningful.
|
||||
"""
|
||||
sent = 0
|
||||
for name, tensor in source:
|
||||
if sent >= len(meta):
|
||||
raise ValueError(
|
||||
f"WeightSource yielded more parameters than metadata() "
|
||||
f"declared ({len(meta)}); first extra is {name!r}."
|
||||
)
|
||||
expected = meta[sent]
|
||||
if (
|
||||
name != expected.name
|
||||
or tensor.dtype != expected.dtype
|
||||
or tuple(tensor.shape) != expected.shape
|
||||
):
|
||||
raise ValueError(
|
||||
"WeightSource metadata() disagrees with iteration at index "
|
||||
f"{sent}: declared {expected.name!r} "
|
||||
f"{expected.dtype} {tuple(expected.shape)}, got {name!r} "
|
||||
f"{tensor.dtype} {tuple(tensor.shape)}. Both channels must "
|
||||
"enumerate the same parameters in the same order."
|
||||
)
|
||||
sent += 1
|
||||
yield name, tensor
|
||||
if sent != len(meta):
|
||||
raise ValueError(
|
||||
f"WeightSource yielded {sent} parameters but metadata() "
|
||||
f"declared {len(meta)}; the worker is waiting for the rest."
|
||||
)
|
||||
|
||||
def _post_send_sync(self) -> None:
|
||||
"""Wait for this rank's transfer work to land before returning.
|
||||
|
||||
Broadcasts are only *enqueued* by `send_weights`: the unpacked path on
|
||||
the current stream, the packed path on the producer's own streams (which
|
||||
it drains itself). Waiting here lets a caller mutate parameters, or
|
||||
start the next step on another stream, as soon as `send_weights`
|
||||
returns, instead of silently depending on same-stream ordering. Every
|
||||
rank waits: a non-sender's `full_tensor()` gathers feed the sender's
|
||||
broadcast, so they must have landed before it may touch its shards.
|
||||
|
||||
Unlike IPC there is no cross-rank barrier here. Nothing in this backend
|
||||
outlives the collective it travelled in (IPC's barrier keeps shared
|
||||
buffers alive until every consumer has opened them), so a barrier would
|
||||
only add a dependency on the default process group that this backend
|
||||
otherwise does not have.
|
||||
"""
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
@@ -1,494 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Packed tensor utilities for efficient weight transfer."""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch.multiprocessing.reductions import reduce_tensor
|
||||
|
||||
# Default values for packed tensor transfer.
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024 # 1GB
|
||||
DEFAULT_PACKED_NUM_BUFFERS = 2
|
||||
|
||||
|
||||
# Streams are cached across calls: the caching allocator keys cached blocks by the
|
||||
# stream they were allocated on, so fresh streams at every call would strand the
|
||||
# packed buffers' reserved memory per call. See gh-52950.
|
||||
@cache
|
||||
def _get_streams(device_idx: int, num_buffers: int) -> tuple[torch.cuda.Stream, ...]:
|
||||
return tuple(torch.cuda.Stream(device=device_idx) for _ in range(num_buffers))
|
||||
|
||||
|
||||
def unpack_tensor(
|
||||
packed_tensor: torch.Tensor,
|
||||
names: list[str],
|
||||
shapes: list[list[int]],
|
||||
dtypes: list[torch.dtype],
|
||||
tensor_sizes: list[int],
|
||||
) -> list[tuple[str, torch.Tensor]]:
|
||||
"""Unpack a packed uint8 tensor into a list of named tensors.
|
||||
|
||||
The returned tensors are **views** of ``packed_tensor`` (the
|
||||
``.contiguous()`` call is a no-op on already-contiguous row-slices).
|
||||
If ``packed_tensor`` lives in storage that may be reused — e.g. a
|
||||
reused CUDA IPC buffer — callers must clone the results before the
|
||||
underlying storage is overwritten.
|
||||
|
||||
Args:
|
||||
packed_tensor: The packed torch.uint8 tensor to unpack
|
||||
names: List of tensor names
|
||||
shapes: List of tensor shapes
|
||||
dtypes: List of tensor dtypes
|
||||
tensor_sizes: List of tensor sizes in bytes
|
||||
|
||||
"""
|
||||
unpacked_tensors = packed_tensor.split(tensor_sizes)
|
||||
|
||||
return [
|
||||
(name, tensor.contiguous().view(dtype).view(shape))
|
||||
for name, shape, dtype, tensor in zip(names, shapes, dtypes, unpacked_tensors)
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PackedChunk:
|
||||
"""Result of packing tensors into a single contiguous uint8 buffer."""
|
||||
|
||||
packed_tensor: torch.Tensor
|
||||
names: list[str]
|
||||
shapes: list[list[int]]
|
||||
dtypes: list[torch.dtype]
|
||||
tensor_sizes: list[int]
|
||||
|
||||
|
||||
def pack_tensors(
|
||||
iterator: Iterator[tuple[str, torch.Tensor]],
|
||||
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
|
||||
buffer_size_bytes: int,
|
||||
tensor_list: list[torch.Tensor] | None = None,
|
||||
current_size: int = 0,
|
||||
) -> PackedChunk | None:
|
||||
"""Pack tensors from an iterator into a single contiguous uint8 buffer.
|
||||
|
||||
Consumes from the iterator until the accumulated size exceeds
|
||||
buffer_size_bytes or the iterator is exhausted, then returns a
|
||||
PackedChunk. Returns None if no tensors were consumed.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of (name, tensor) pairs
|
||||
post_iter_func: Applied to each item before linearizing to uint8
|
||||
buffer_size_bytes: Max bytes before flushing
|
||||
tensor_list: Pre-existing tensor list to append to (for NCCL
|
||||
multi-buffer reuse). If None, a fresh list is created.
|
||||
current_size: Byte count already accumulated in tensor_list
|
||||
|
||||
"""
|
||||
if tensor_list is None:
|
||||
tensor_list = []
|
||||
|
||||
names: list[str] = []
|
||||
shapes: list[list[int]] = []
|
||||
dtypes: list[torch.dtype] = []
|
||||
tensor_sizes: list[int] = []
|
||||
total_bytes = current_size
|
||||
|
||||
while True:
|
||||
try:
|
||||
item = next(iterator)
|
||||
except StopIteration:
|
||||
break
|
||||
|
||||
name, orig_tensor = item
|
||||
# Apply post processing and convert to linearized uint8 tensor
|
||||
tensor = post_iter_func(item).contiguous().view(-1).view(torch.uint8)
|
||||
|
||||
if tensor.numel() > buffer_size_bytes:
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
f"Tensor '{name}' has size {tensor.numel()} bytes, which "
|
||||
f"exceeds buffer_size_bytes={buffer_size_bytes}.",
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
tensor_list.append(tensor)
|
||||
names.append(name)
|
||||
shapes.append(list(orig_tensor.shape))
|
||||
dtypes.append(orig_tensor.dtype)
|
||||
tensor_sizes.append(tensor.numel())
|
||||
total_bytes += tensor.numel()
|
||||
|
||||
if total_bytes > buffer_size_bytes:
|
||||
break
|
||||
|
||||
if not tensor_list:
|
||||
return None
|
||||
|
||||
packed = torch.cat(tensor_list, dim=0)
|
||||
del tensor_list
|
||||
return PackedChunk(
|
||||
packed_tensor=packed,
|
||||
names=names,
|
||||
shapes=shapes,
|
||||
dtypes=dtypes,
|
||||
tensor_sizes=tensor_sizes,
|
||||
)
|
||||
|
||||
|
||||
# ── NCCL packed broadcast ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def packed_nccl_broadcast_producer(
|
||||
iterator: Iterator[tuple[str, torch.Tensor]],
|
||||
group: Any,
|
||||
src: int,
|
||||
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
|
||||
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
|
||||
) -> None:
|
||||
"""Broadcast tensors in a packed manner from trainer to workers.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of model parameters. Returns a tuple of (name, tensor)
|
||||
group: Process group (PyNcclCommunicator)
|
||||
src: Source rank (0 in current implementation)
|
||||
post_iter_func: Function to apply to each (name, tensor) pair before
|
||||
packing, should return a tensor
|
||||
buffer_size_bytes: Size in bytes for each packed tensor buffer.
|
||||
Both producer and consumer must use the same value.
|
||||
num_buffers: Number of buffers for double/triple buffering.
|
||||
Both producer and consumer must use the same value.
|
||||
|
||||
"""
|
||||
streams = _get_streams(torch.accelerator.current_device_index(), num_buffers)
|
||||
# Keep references to in-flight chunks so their packed_tensors
|
||||
# aren't freed while an async broadcast is still reading them.
|
||||
in_flight: list[PackedChunk | None] = [None] * num_buffers
|
||||
buffer_idx = 0
|
||||
|
||||
while True:
|
||||
# Synchronize the current stream
|
||||
streams[buffer_idx].synchronize()
|
||||
# Previous chunk on this buffer slot is now safe to free
|
||||
in_flight[buffer_idx] = None
|
||||
# Start tasks for the new buffer in a new stream
|
||||
with torch.cuda.stream(streams[buffer_idx]):
|
||||
chunk = pack_tensors(iterator, post_iter_func, buffer_size_bytes)
|
||||
if chunk is None:
|
||||
break
|
||||
# Pack the tensors and call broadcast collective
|
||||
group.broadcast(chunk.packed_tensor, src=src)
|
||||
# Hold reference until this stream is synchronized
|
||||
in_flight[buffer_idx] = chunk
|
||||
# Move to the next buffer
|
||||
buffer_idx = (buffer_idx + 1) % num_buffers
|
||||
|
||||
# Drain every slot before returning.
|
||||
for stream in streams:
|
||||
stream.synchronize()
|
||||
|
||||
|
||||
def packed_nccl_broadcast_consumer(
|
||||
iterator: Iterator[tuple[str, tuple[list[int], torch.dtype]]],
|
||||
group: Any,
|
||||
src: int,
|
||||
post_unpack_func: Callable[[list[tuple[str, torch.Tensor]]], None],
|
||||
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
|
||||
device: torch.device | str = "cuda",
|
||||
) -> None:
|
||||
"""Consume packed tensors and unpack them into a list of tensors.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of parameter metadata. Returns (name, (shape, dtype))
|
||||
group: Process group (PyNcclCommunicator)
|
||||
src: Source rank (0 in current implementation)
|
||||
post_unpack_func: Function to apply to each list of (name, tensor) after
|
||||
unpacking
|
||||
buffer_size_bytes: Size in bytes for each packed tensor buffer.
|
||||
Both producer and consumer must use the same value.
|
||||
num_buffers: Number of buffers for double/triple buffering.
|
||||
Both producer and consumer must use the same value.
|
||||
device: Device for the receive buffers. Must match the device the NCCL
|
||||
communicator was created on (the worker's assigned device).
|
||||
|
||||
"""
|
||||
target_packed_tensor_size = buffer_size_bytes
|
||||
|
||||
streams = _get_streams(torch.accelerator.current_device_index(), num_buffers)
|
||||
buffer_idx = 0
|
||||
|
||||
packing_tensor_meta_data: list[list[tuple[str, list[int], torch.dtype, int]]] = [
|
||||
[] for _ in range(num_buffers)
|
||||
]
|
||||
packing_tensor_sizes: list[int] = [0 for _ in range(num_buffers)]
|
||||
packed_tensors: list[torch.Tensor] = [
|
||||
torch.empty(0, dtype=torch.uint8, device=device) for _ in range(num_buffers)
|
||||
]
|
||||
|
||||
while True:
|
||||
# Synchronize the current stream
|
||||
streams[buffer_idx].synchronize()
|
||||
with torch.cuda.stream(streams[buffer_idx]):
|
||||
# Initialize the packing tensor meta data
|
||||
packing_tensor_meta_data[buffer_idx] = []
|
||||
packing_tensor_sizes[buffer_idx] = 0
|
||||
try:
|
||||
# Form a packed tensor
|
||||
while True:
|
||||
name, (shape, dtype) = next(iterator)
|
||||
tensor_size = math.prod(shape) * dtype.itemsize
|
||||
packing_tensor_meta_data[buffer_idx].append(
|
||||
(name, shape, dtype, tensor_size)
|
||||
)
|
||||
packing_tensor_sizes[buffer_idx] += tensor_size
|
||||
if packing_tensor_sizes[buffer_idx] > target_packed_tensor_size:
|
||||
break
|
||||
# Create a packed tensor and broadcast it
|
||||
packed_tensors[buffer_idx] = torch.empty(
|
||||
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device=device
|
||||
)
|
||||
group.broadcast(packed_tensors[buffer_idx], src=src)
|
||||
# Load the packed tensor into the model
|
||||
names, shapes, dtypes, tensor_sizes = zip(
|
||||
*packing_tensor_meta_data[buffer_idx]
|
||||
)
|
||||
post_unpack_func(
|
||||
unpack_tensor(
|
||||
packed_tensors[buffer_idx],
|
||||
list(names),
|
||||
list(shapes),
|
||||
list(dtypes),
|
||||
list(tensor_sizes),
|
||||
)
|
||||
)
|
||||
# Move to the next buffer
|
||||
buffer_idx = (buffer_idx + 1) % num_buffers
|
||||
except StopIteration:
|
||||
# Do the last broadcast if there are remaining tensors
|
||||
if len(packing_tensor_meta_data[buffer_idx]) > 0:
|
||||
# Create a packed tensor and broadcast it
|
||||
packed_tensors[buffer_idx] = torch.empty(
|
||||
packing_tensor_sizes[buffer_idx],
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
group.broadcast(packed_tensors[buffer_idx], src=src)
|
||||
# Load the packed tensor into the model
|
||||
names, shapes, dtypes, tensor_sizes = zip(
|
||||
*packing_tensor_meta_data[buffer_idx]
|
||||
)
|
||||
post_unpack_func(
|
||||
unpack_tensor(
|
||||
packed_tensors[buffer_idx],
|
||||
list(names),
|
||||
list(shapes),
|
||||
list(dtypes),
|
||||
list(tensor_sizes),
|
||||
)
|
||||
)
|
||||
break
|
||||
|
||||
# Drain every slot before returning.
|
||||
for stream in streams:
|
||||
stream.synchronize()
|
||||
|
||||
|
||||
# ── IPC packed transfer ────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class PackedIpcChunk:
|
||||
"""Metadata and IPC handle for a single packed chunk."""
|
||||
|
||||
names: list[str]
|
||||
shapes: list[list[int]]
|
||||
dtype_names: list[str]
|
||||
tensor_sizes: list[int]
|
||||
ipc_handle: dict[str, tuple]
|
||||
|
||||
|
||||
def packed_ipc_producer(
|
||||
iterator: Iterator[tuple[str, torch.Tensor]],
|
||||
gpu_uuid: str,
|
||||
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
|
||||
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
) -> Iterator[PackedIpcChunk]:
|
||||
"""Pack tensors into a reusable IPC buffer and yield handles.
|
||||
|
||||
Allocates a single GPU buffer of ``buffer_size_bytes`` and registers
|
||||
it for IPC once via ``reduce_tensor``. Each chunk's packed data is
|
||||
copied into this buffer before yielding, so only one IPC-shared
|
||||
allocation exists for the lifetime of the transfer.
|
||||
|
||||
Callers **must** ensure the consumer has finished reading the buffer
|
||||
(e.g. ``ray.get`` returned) before resuming the generator for the
|
||||
next chunk.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of (name, tensor) pairs.
|
||||
gpu_uuid: Physical GPU UUID string for this rank.
|
||||
post_iter_func: Applied to each (name, tensor) before packing.
|
||||
buffer_size_bytes: Exact capacity of the reusable IPC buffer.
|
||||
Every chunk is guaranteed to fit within this size. A
|
||||
``ValueError`` is raised if any single tensor exceeds it.
|
||||
|
||||
"""
|
||||
ipc_buffer = torch.empty(buffer_size_bytes, dtype=torch.uint8, device="cuda")
|
||||
_, ipc_args = reduce_tensor(ipc_buffer)
|
||||
|
||||
names: list[str] = []
|
||||
shapes: list[list[int]] = []
|
||||
dtypes: list[torch.dtype] = []
|
||||
tensor_sizes: list[int] = []
|
||||
total_bytes = 0
|
||||
|
||||
for name, orig_tensor in iterator:
|
||||
flat = (
|
||||
post_iter_func((name, orig_tensor)).contiguous().view(-1).view(torch.uint8)
|
||||
)
|
||||
|
||||
if flat.numel() > buffer_size_bytes:
|
||||
raise ValueError(
|
||||
f"Tensor '{name}' has size {flat.numel()} bytes, "
|
||||
f"which exceeds buffer_size_bytes={buffer_size_bytes}. "
|
||||
f"Increase buffer_size_bytes to at least {flat.numel()}."
|
||||
)
|
||||
|
||||
if total_bytes and total_bytes + flat.numel() > buffer_size_bytes:
|
||||
# Drain queued copies so the consumer sees a fully-written buffer.
|
||||
torch.cuda.current_stream().synchronize()
|
||||
yield PackedIpcChunk(
|
||||
names=names,
|
||||
shapes=shapes,
|
||||
dtype_names=[str(d).split(".")[-1] for d in dtypes],
|
||||
tensor_sizes=tensor_sizes,
|
||||
ipc_handle={gpu_uuid: ipc_args},
|
||||
)
|
||||
# Rebind to fresh lists so the yielded chunk's metadata is
|
||||
# not mutated while the consumer is still reading.
|
||||
names, shapes, dtypes, tensor_sizes = [], [], [], []
|
||||
total_bytes = 0
|
||||
|
||||
ipc_buffer[total_bytes : total_bytes + flat.numel()].copy_(flat)
|
||||
names.append(name)
|
||||
shapes.append(list(orig_tensor.shape))
|
||||
dtypes.append(orig_tensor.dtype)
|
||||
tensor_sizes.append(flat.numel())
|
||||
total_bytes += flat.numel()
|
||||
|
||||
if total_bytes:
|
||||
torch.cuda.current_stream().synchronize()
|
||||
yield PackedIpcChunk(
|
||||
names=names,
|
||||
shapes=shapes,
|
||||
dtype_names=[str(d).split(".")[-1] for d in dtypes],
|
||||
tensor_sizes=tensor_sizes,
|
||||
ipc_handle={gpu_uuid: ipc_args},
|
||||
)
|
||||
|
||||
|
||||
class PackedBufferImporter:
|
||||
"""One-slot cache for the consumer-side mapping of a packed IPC buffer.
|
||||
|
||||
torch's cross-process refcount pairs one ``reduce_tensor`` export with
|
||||
one consumer rebuild-release cycle. The producer exports its buffer once
|
||||
and ships the same args with every chunk, so rebuilding and releasing
|
||||
per chunk decrements the counter once per chunk, drives it negative,
|
||||
and the dropped buffer is then never reclaimable by ``ipc_collect`` on
|
||||
the producer side (it leaks one buffer per transfer). Caching the
|
||||
rebuilt buffer keyed by the rebuild args restores the pairing: chunks
|
||||
of one transfer reuse the mapping (it aliases producer memory, so it
|
||||
always reads the current chunk's bytes), and replacing the entry when
|
||||
the next export arrives — or ``close()`` — releases the previous
|
||||
mapping exactly once.
|
||||
|
||||
The importer that consumes a multi-chunk transfer must be one object
|
||||
reused across chunks; a fresh importer per chunk reintroduces the
|
||||
per-chunk release. ``IPCWeightTransferEngine`` owns one instance per
|
||||
engine and closes it on shutdown.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._entry: tuple[tuple, torch.Tensor] | None = None
|
||||
|
||||
def rebuild(self, list_args: list) -> torch.Tensor:
|
||||
from torch.multiprocessing.reductions import rebuild_cuda_tensor
|
||||
|
||||
key = tuple(list_args)
|
||||
if self._entry is not None and self._entry[0] == key:
|
||||
return self._entry[1]
|
||||
packed = rebuild_cuda_tensor(*list_args)
|
||||
self._entry = (key, packed)
|
||||
return packed
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release the held mapping (one producer-side refcount decrement)."""
|
||||
self._entry = None
|
||||
|
||||
|
||||
def packed_ipc_consumer(
|
||||
ipc_handle: dict[str, tuple],
|
||||
names: list[str],
|
||||
shapes: list[list[int]],
|
||||
dtype_names: list[str],
|
||||
tensor_sizes: list[int],
|
||||
device_index: int,
|
||||
importer: "PackedBufferImporter | None" = None,
|
||||
) -> list[tuple[str, torch.Tensor]]:
|
||||
"""Unpack a single packed IPC chunk into named tensors.
|
||||
|
||||
Reconstructs the packed buffer via rebuild_cuda_tensor, unpacks
|
||||
into individual tensors, and clones each into independent storage
|
||||
before returning.
|
||||
|
||||
The clone is intentional: the producer reuses one IPC buffer across
|
||||
chunks, so any tensor view that aliases the buffer would observe the
|
||||
*next* chunk's bytes as soon as the producer's generator is resumed.
|
||||
Callers that retain references past their own update_weights call
|
||||
(notably vLLM's layerwise reload, which buffers ``bound_args`` for
|
||||
replay in ``_layerwise_process``) would otherwise replay against
|
||||
stale data and silently corrupt multi-chunk weight transfers.
|
||||
|
||||
Args:
|
||||
ipc_handle: Mapping of GPU UUID to rebuild_cuda_tensor args tuple
|
||||
names: Parameter names in the packed buffer
|
||||
shapes: Parameter shapes
|
||||
dtype_names: Parameter dtype name strings (e.g. "float16")
|
||||
tensor_sizes: Size in bytes of each parameter in the packed buffer
|
||||
device_index: Local CUDA device index
|
||||
importer: Import cache that must be shared across all chunks of one
|
||||
export so the buffer mapping is opened and released exactly once
|
||||
(see ``PackedBufferImporter``). ``None`` builds an ephemeral one,
|
||||
which is only safe for single-chunk consumption.
|
||||
|
||||
"""
|
||||
props = torch.cuda.get_device_properties(device_index)
|
||||
physical_gpu_id = str(props.uuid)
|
||||
|
||||
if physical_gpu_id not in ipc_handle:
|
||||
raise ValueError(
|
||||
f"IPC handle not found for GPU UUID {physical_gpu_id}. "
|
||||
f"Available UUIDs: {list(ipc_handle.keys())}"
|
||||
)
|
||||
|
||||
args = ipc_handle[physical_gpu_id]
|
||||
list_args = list(args)
|
||||
list_args[6] = device_index
|
||||
if importer is None:
|
||||
importer = PackedBufferImporter()
|
||||
packed = importer.rebuild(list_args)
|
||||
|
||||
content_size = sum(tensor_sizes)
|
||||
packed = packed[:content_size]
|
||||
|
||||
dtypes = [getattr(torch, dn) for dn in dtype_names]
|
||||
return [
|
||||
(name, t.clone())
|
||||
for name, t in unpack_tensor(packed, names, shapes, dtypes, tensor_sizes)
|
||||
]
|
||||
@@ -1,352 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared pieces of the sharded-RDT backend: the op-chain allowlist, buffer
|
||||
sizing and the minimum Ray version, plus ``RdtRouter`` — consumer-only, since
|
||||
routing is a consumer decision.
|
||||
|
||||
The gather-group partition itself is ``base.layerwise_groups``: it defines what a
|
||||
group index means for any ``WeightSource``, not just this transport.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
# The op-chain contract, with two enforcers. The consumer's ``FakeRDTTensor``
|
||||
# intercepts exactly these methods during the bake and records the string name;
|
||||
# anything else reaches ``__torch_dispatch__`` and raises, so a loader needing
|
||||
# real data fails at init rather than transferring wrong bytes. The producer
|
||||
# replays a chain as ``getattr(tensor, op)(...)`` and refuses anything outside
|
||||
# ``ALLOWED_OPS``, so a spoofed consumer cannot invoke arbitrary methods.
|
||||
#
|
||||
# Every entry must be a pure view; ``to`` in particular must never be added, as
|
||||
# it is the dtype/device escape the bake exists to reject. Both sets derive from
|
||||
# this one table so they cannot disagree.
|
||||
SUPPORTED_OPS: dict[Callable, str] = {
|
||||
torch.Tensor.narrow: "narrow",
|
||||
torch.Tensor.view: "view",
|
||||
torch.Tensor.reshape: "reshape",
|
||||
torch.Tensor.__getitem__: "__getitem__",
|
||||
torch.Tensor.unsqueeze: "unsqueeze",
|
||||
torch.Tensor.squeeze: "squeeze",
|
||||
torch.Tensor.transpose: "transpose",
|
||||
torch.Tensor.t: "t",
|
||||
torch.Tensor.permute: "permute",
|
||||
torch.Tensor.flatten: "flatten",
|
||||
torch.Tensor.contiguous: "contiguous",
|
||||
torch.Tensor.chunk: "chunk",
|
||||
# Multi-return like chunk: one child per output, trailing __getitem__(i).
|
||||
torch.Tensor.unbind: "unbind",
|
||||
}
|
||||
|
||||
ALLOWED_OPS = frozenset(SUPPORTED_OPS.values())
|
||||
|
||||
# Ray Direct Transport landed in halves: the actor opt-in
|
||||
# ``enable_tensor_transport`` in 2.49, and ``ray.experimental``'s
|
||||
# ``register_nixl_memory`` / ``set_target_for_ref`` — which this transport calls
|
||||
# directly — in 2.55. The floor is the tested version rather than the earliest
|
||||
# one that could import; 2.55 has never been run against.
|
||||
RDT_MIN_RAY_VERSION = "2.56.0"
|
||||
|
||||
|
||||
def check_ray_rdt_version() -> None:
|
||||
"""Refuse an installed Ray older than the one this backend is tested on.
|
||||
|
||||
vLLM does not depend on Ray, so there is no pin to carry this. Without the
|
||||
check the failure is an opaque option-validation error out of
|
||||
``.options(enable_tensor_transport=True)`` (below 2.49) or an ImportError
|
||||
raised deep in the first pull, long after init reported success (below 2.55).
|
||||
|
||||
Raises:
|
||||
ValueError: the installed Ray predates ``RDT_MIN_RAY_VERSION``.
|
||||
|
||||
"""
|
||||
import importlib.metadata
|
||||
|
||||
from packaging import version
|
||||
|
||||
required = version.parse(RDT_MIN_RAY_VERSION)
|
||||
current = version.parse(importlib.metadata.version("ray"))
|
||||
if current < required:
|
||||
raise ValueError(
|
||||
f"The 'sharded_rdt' weight transfer backend requires Ray "
|
||||
f">= {required}, the version it is tested against; Ray Direct "
|
||||
f"Transport (ray.experimental.register_nixl_memory / "
|
||||
f"set_target_for_ref) needs at least 2.55. Found {current}. "
|
||||
f"Run `pip install -U 'ray>={required}'`."
|
||||
)
|
||||
|
||||
|
||||
def register_nixl_memory(tensor: torch.Tensor) -> None:
|
||||
"""Register a tensor with Ray using the platform's NIXL package."""
|
||||
from vllm.distributed.nixl_utils import alias_nixl_for_ray
|
||||
|
||||
alias_nixl_for_ray()
|
||||
from ray.experimental import register_nixl_memory as ray_register_nixl_memory
|
||||
|
||||
ray_register_nixl_memory(tensor)
|
||||
|
||||
|
||||
def assign_producer_indices(
|
||||
num_producers: int, num_consumers: int, consumer_idx: int
|
||||
) -> list[int]:
|
||||
"""Producers (global indices) that consumer ``consumer_idx`` binds."""
|
||||
p = max(1, num_producers)
|
||||
c = max(1, num_consumers)
|
||||
if p >= c:
|
||||
return list(range(consumer_idx * p // c, (consumer_idx + 1) * p // c))
|
||||
return [consumer_idx * p // c]
|
||||
|
||||
|
||||
class RdtRouter:
|
||||
"""Decides which producer serves each weight name, and — once bound — sends.
|
||||
|
||||
Ownership is per NAME. ``owner_sets`` holds the few distinct producer sets
|
||||
that occur, and ``name_owner_class[i]`` indexes it for ``names[i]``; a name
|
||||
determines both its owner set and its gather group, so neither appears in the
|
||||
routing API. Empty tables mean every producer holds everything.
|
||||
|
||||
Any placement is expressible this way: a pipeline stage is a set of names
|
||||
sharing one owner set, an expert is a name whose owner set is a single rank,
|
||||
and a group produced by two stages is just two classes inside one group.
|
||||
|
||||
Both engines derive the same tables from the same wire data, so they agree on
|
||||
who serves what. Disagreement is not a wrong answer but a hang or a loud
|
||||
misroute: a pull sent to a producer that never gathered the name trips its
|
||||
served-names guard.
|
||||
|
||||
Routing is consumer-only. The trainer publishes what it holds and answers
|
||||
whatever arrives; it never asks who serves what.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_producers: int,
|
||||
num_consumers: int,
|
||||
owner_sets: list[list[int]] | None = None,
|
||||
name_owner_class: list[int] | None = None,
|
||||
names: list[str] | None = None,
|
||||
group_lens: list[int] | None = None,
|
||||
workers_per_replica: int = 0,
|
||||
) -> None:
|
||||
"""Build the routing tables from the wire data both engines receive.
|
||||
|
||||
The four table arguments travel together and default together: with all
|
||||
of them empty the router degrades to "every producer holds everything".
|
||||
|
||||
Args:
|
||||
num_producers: Trainer ranks. Owner indices are positions in this
|
||||
range, and ``validate`` rejects any that fall outside it.
|
||||
num_consumers: Inference workers across the whole fleet. Fixes
|
||||
the id space; the block carve uses ``workers_per_replica``
|
||||
of it.
|
||||
owner_sets: The distinct producer sets that occur, one row per
|
||||
owner class; each row is deduplicated and sorted here. Empty
|
||||
means a single class owning every producer.
|
||||
name_owner_class: Parallel to ``names`` — ``name_owner_class[i]``
|
||||
indexes ``owner_sets`` for ``names[i]``. A name with no entry
|
||||
falls back to class 0.
|
||||
names: Every parameter name, in group-major order: concatenating
|
||||
the gather groups reproduces this list exactly. The other two
|
||||
tables are keyed by this order.
|
||||
group_lens: Length of each gather group, consecutive over
|
||||
``names``, so they sum to ``len(names)``. Fixes name -> group
|
||||
and the per-group owner union the free barrier fans out to.
|
||||
workers_per_replica: Consumers per inference DEPLOYMENT. The block
|
||||
carve spreads this many consumers over an owner set and every
|
||||
deployment reuses that carve, so the same worker of each
|
||||
deployment resolves one producer (see ``producer_for``). 0
|
||||
means one deployment: carve over the whole fleet.
|
||||
|
||||
Raises:
|
||||
ValueError: ``workers_per_replica`` does not divide
|
||||
``num_consumers``, so the deployments are not uniform and two
|
||||
workers of one deployment would share a block index.
|
||||
|
||||
"""
|
||||
self.num_producers = max(1, num_producers)
|
||||
self.num_consumers = max(1, num_consumers)
|
||||
# Width of the block carve: one deployment's worth of consumers, so
|
||||
# deployments overlay rather than spread. See ``producer_for``.
|
||||
self._block_consumers = min(
|
||||
self.num_consumers, max(1, workers_per_replica or self.num_consumers)
|
||||
)
|
||||
if self.num_consumers % self._block_consumers:
|
||||
raise ValueError(
|
||||
f"workers_per_replica={workers_per_replica} does not divide "
|
||||
f"num_consumers={self.num_consumers}; the inference deployments "
|
||||
f"must be uniform for the per-deployment block carve to be "
|
||||
f"well defined."
|
||||
)
|
||||
self._owner_sets = (
|
||||
[sorted(set(row)) for row in owner_sets]
|
||||
if owner_sets
|
||||
else [list(range(self.num_producers))]
|
||||
)
|
||||
names = list(names or [])
|
||||
classes = list(name_owner_class or [])
|
||||
self._class_of = {
|
||||
n: (classes[i] if i < len(classes) else 0) for i, n in enumerate(names)
|
||||
}
|
||||
# name -> gather group, and the per-group owner union the free barrier
|
||||
# fans out to. Both are pure functions of the tables, so precompute once.
|
||||
self._group_of: dict[str, int] = {}
|
||||
self._group_owners: list[list[int]] = []
|
||||
pos = 0
|
||||
for gi, glen in enumerate(group_lens or []):
|
||||
union: set[int] = set()
|
||||
for n in names[pos : pos + glen]:
|
||||
self._group_of[n] = gi
|
||||
c = self._class_of.get(n, 0)
|
||||
# Bounds-checked: a bad class must surface from validate() as a
|
||||
# named ValueError, not as an IndexError from this precompute.
|
||||
if 0 <= c < len(self._owner_sets):
|
||||
union.update(self._owner_sets[c])
|
||||
pos += glen
|
||||
self._group_owners.append(sorted(union))
|
||||
# Bound at init on the consumer; the rule methods never touch these.
|
||||
self._actors: list[Any] = []
|
||||
self._produce_methods: list[Any] = []
|
||||
self.consumer_id = 0
|
||||
|
||||
# ---------------- rules (pure; safe to use unbound) ----------------
|
||||
|
||||
def class_of(self, name: str) -> int:
|
||||
"""The name's owner class — the planner's bucketing key, since all names
|
||||
of a chunk must share one producer."""
|
||||
return self._class_of.get(name, 0)
|
||||
|
||||
def owners(self, name: str) -> list[int]:
|
||||
"""Every producer holding ``name``."""
|
||||
return list(self._owner_sets[self.class_of(name)])
|
||||
|
||||
def group_owners(self, group_idx: int) -> list[int]:
|
||||
"""Every producer holding any name of ``group_idx``.
|
||||
|
||||
The one group-keyed rule, because the free barrier is per group: a
|
||||
consumer signals ``free_group(gi)`` at each of these and the producer
|
||||
counts signals against its live-consumer total. Names route; groups free.
|
||||
"""
|
||||
if not self._group_owners:
|
||||
return list(range(self.num_producers))
|
||||
return list(self._group_owners[group_idx])
|
||||
|
||||
def producer_for(self, consumer_id: int, name: str) -> int:
|
||||
"""The single producer ``consumer_id`` pulls ``name`` from.
|
||||
|
||||
Blocks consumers across the name's owner set with the same rule that
|
||||
binds producers globally, then rotates by the name's GROUP index so a
|
||||
consumer spreads its groups over its block instead of hammering one NIC.
|
||||
Rotating per group (not per name) keeps every name of a chunk on one
|
||||
producer, which is what lets a chunk be a single pull.
|
||||
|
||||
The carve is over ONE DEPLOYMENT (this consumer's index within its own
|
||||
deployment, over ``workers_per_replica`` of them), so the same worker of
|
||||
every deployment resolves the same producer for every name. Carving over
|
||||
the whole fleet instead spreads the multi-owner names of different
|
||||
deployments onto different producers while single-owner names still
|
||||
route by ownership, so a producer serves several distinct worker
|
||||
indices, each needing its own serve ring, and none of them can share a
|
||||
slot. With one deployment the width is the fleet, so this is the plain
|
||||
block rule.
|
||||
"""
|
||||
own = self.owners(name)
|
||||
if not own:
|
||||
raise ValueError(f"{name!r} has no owner")
|
||||
block = assign_producer_indices(
|
||||
len(own), self._block_consumers, consumer_id % self._block_consumers
|
||||
)
|
||||
return own[block[self._group_of.get(name, 0) % len(block)]]
|
||||
|
||||
def validate(self) -> None:
|
||||
"""Check the ownership tables can be served.
|
||||
|
||||
Raises:
|
||||
ValueError: an owner set is empty or out of range, or a class index
|
||||
does not resolve.
|
||||
|
||||
"""
|
||||
for c, row in enumerate(self._owner_sets):
|
||||
if not row:
|
||||
raise ValueError(f"owner set {c} is empty: no producer holds it")
|
||||
bad = [p for p in row if not 0 <= p < self.num_producers]
|
||||
if bad:
|
||||
raise ValueError(f"owner set {c} out of range: {bad}")
|
||||
for name, c in self._class_of.items():
|
||||
if not 0 <= c < len(self._owner_sets):
|
||||
raise ValueError(
|
||||
f"{name!r} has owner class {c}, but only "
|
||||
f"{len(self._owner_sets)} owner set(s) were shipped"
|
||||
)
|
||||
|
||||
# ---------------- send surface (needs the binding) ----------------
|
||||
|
||||
def bind(self, actors: list, produce_methods: list, consumer_id: int) -> None:
|
||||
"""Attach this consumer's producer handles. Rebuilt wholesale per init,
|
||||
never appended to: every owner index is a position in these lists, so a
|
||||
rejoining engine that re-inits must not shift them."""
|
||||
if len(actors) != len(produce_methods):
|
||||
raise ValueError("actors and produce_methods must be parallel")
|
||||
self._actors = list(actors)
|
||||
self._produce_methods = list(produce_methods)
|
||||
self.consumer_id = consumer_id
|
||||
|
||||
def _bound(self) -> None:
|
||||
if not self._produce_methods:
|
||||
raise RuntimeError("RdtRouter has no producers bound; call bind() first.")
|
||||
|
||||
def pull(self, owner: int, keys: list, seq: int) -> Any:
|
||||
"""Issue one packed pull to ``owner``.
|
||||
|
||||
The consumer id keys the producer's serve ring; without it every worker
|
||||
is served out of ring 0 and concurrent pulls overwrite each other's blob.
|
||||
``seq`` is this call's index in the stream to ``owner`` (see ``_Chunk``):
|
||||
it selects the serve slot in ISSUE order, so a slot is never repacked
|
||||
while the read of its previous contents is still in flight.
|
||||
"""
|
||||
self._bound()
|
||||
return self._produce_methods[owner].remote(
|
||||
keys, consumer_id=self.consumer_id, seq=seq
|
||||
)
|
||||
|
||||
def free_group(self, group_idx: int) -> list:
|
||||
"""Signal the group done at every producer holding any of its names."""
|
||||
self._bound()
|
||||
return [
|
||||
self._actors[p].free_group.remote(group_idx)
|
||||
for p in self.group_owners(group_idx)
|
||||
]
|
||||
|
||||
def reserve_serve_buffers(
|
||||
self, bytes_by_producer: list[int], plan_digests: list[str] | None = None
|
||||
) -> list:
|
||||
"""Ask each producer to pre-register a serve ring sized to the most this
|
||||
consumer will pull from it.
|
||||
|
||||
``plan_digests[p]`` describes the chunks this consumer pulls from
|
||||
producer ``p``, in pull order. A producer that shares one serve ring
|
||||
across the consumers of several deployments compares it across them, so
|
||||
a fleet whose deployments are not identical fails at init instead of
|
||||
stalling mid-sync. A producer that shares nothing ignores it.
|
||||
"""
|
||||
self._bound()
|
||||
digests: list = (
|
||||
list(plan_digests) if plan_digests else [None] * len(bytes_by_producer)
|
||||
)
|
||||
return [
|
||||
self._actors[p].reserve_serve_buffer.remote(
|
||||
self.consumer_id, nb, digests[p]
|
||||
)
|
||||
for p, nb in enumerate(bytes_by_producer)
|
||||
if nb > 0
|
||||
]
|
||||
|
||||
|
||||
def buffer_alloc_bytes(nbytes: int, presize: int = 0) -> int:
|
||||
"""Size a NIXL buffer / ring slot for ``nbytes``: the max of the request, an
|
||||
optional ``presize`` floor, and a coarse 256MB round-up, so the buffer is
|
||||
allocated ONCE and never regrows. Regrowth is a correctness hazard, not just a
|
||||
perf one -- see ``buffer_presize_gb``. Shared by the consumer's receive buffers
|
||||
and the producer's serve rings."""
|
||||
return max(nbytes, presize, -(-nbytes // (256 << 20)) * (256 << 20))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,300 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Op-chain recording for the sharded-RDT backend.
|
||||
|
||||
The consumer asks the trainer for the exact slice a worker consumes, described as
|
||||
an op chain replayed on the trainer's live tensor. ``FakeRDTTensor`` builds the
|
||||
chain by intercepting the model's own weight loaders; ``copy_`` is its data sink,
|
||||
``BakeSink`` — the dry run records how each slice would be fetched and where it
|
||||
lands.
|
||||
|
||||
Chains are built against meta tensors, so nothing here touches Ray or a GPU.
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_common import SUPPORTED_OPS
|
||||
|
||||
# ("op_name", positional_args, sorted_kwargs_items). Entries must be hashable so
|
||||
# the chain can serve as a FetchKey.
|
||||
OpSpec = tuple[str, tuple[Any, ...], tuple[tuple[str, Any], ...]]
|
||||
OpChain = tuple[OpSpec, ...]
|
||||
FetchKey = tuple[str, OpChain]
|
||||
|
||||
|
||||
def _freeze_kwargs(kwargs: dict[str, Any]) -> tuple[tuple[str, Any], ...]:
|
||||
"""Sort kwargs into a tuple of items for hashable storage in OpSpec."""
|
||||
return tuple(sorted(kwargs.items()))
|
||||
|
||||
|
||||
class _UnsupportedFakeOp(NotImplementedError):
|
||||
"""Raised when a weight loader does something to a FakeRDTTensor that cannot
|
||||
be expressed as a slice request.
|
||||
|
||||
Surfaced as NotImplementedError so callers can distinguish "this backend
|
||||
can't handle this loader" from genuine bugs.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Scatter:
|
||||
"""One recorded scatter: pull ``src``, copy it into ``layer``'s
|
||||
``param_name`` at the recorded strided region. The bake's output unit; see
|
||||
the engine module's Data flow.
|
||||
|
||||
Self-contained so replay needs no lookups. Destination geometry is read off
|
||||
the meta view and rebuilt each sync as ``as_strided(shape, stride, offset)``
|
||||
-- ``layer`` is resolved to a param at replay time, never baked, since every
|
||||
sync re-materializes fresh tensors.
|
||||
|
||||
``shape`` and ``dtype`` also size the slice ON THE WIRE, so ``dtype`` is the
|
||||
PRODUCED dtype (the fake's after its op chain), not the source name's:
|
||||
``view(dtype)`` is allowlisted, and taking the source's would size the slice
|
||||
with the wrong itemsize and carve the packed blob differently on the two
|
||||
sides.
|
||||
"""
|
||||
|
||||
layer: Any
|
||||
param_name: str
|
||||
src: FetchKey
|
||||
offset: int
|
||||
shape: tuple[int, ...]
|
||||
stride: tuple[int, ...]
|
||||
dtype: torch.dtype
|
||||
|
||||
|
||||
def _meta_copy_(dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor:
|
||||
"""Fire ``dest.copy_`` from a zero-storage meta source of ``src``'s geometry.
|
||||
|
||||
Moves no data but still counts against the layer's loaded numel, which drives
|
||||
``_layerwise_process``; skipping it would leave the layer looking unloaded
|
||||
forever.
|
||||
|
||||
A non-meta ``dest`` is one the layerwise reload never moved there (the
|
||||
``SKIP_LOAD_TENSORS`` set, e.g. GLM's router bias). Torch forbids
|
||||
``real.copy_(meta)``, and skipping the count is consistent -- ``get_layer_size``
|
||||
excludes the same set. The caller still RECORDS the copy, so the replay writes
|
||||
the real param.
|
||||
"""
|
||||
if not dest.is_meta:
|
||||
return dest
|
||||
meta_src = torch.empty(src.shape, dtype=src.dtype, device="meta")
|
||||
with torch._C.DisableTorchFunctionSubclass():
|
||||
return dest.copy_(meta_src)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BakeSink:
|
||||
"""``copy_`` sink for the dry-run bake: record how each slice would be fetched
|
||||
and where it would land, and move nothing.
|
||||
|
||||
``_install_recording_stamps`` sets ``current = (leaf_module, param_name)``
|
||||
around each loader so ``accept_copy`` can attribute the copy. An unstamped
|
||||
``copy_`` cannot be attributed and stays unrecorded, so its module fails the
|
||||
coverage gate and takes the plain load. ``copies_by_layer`` is keyed by module
|
||||
object, so iterating it yields each leaf module once.
|
||||
"""
|
||||
|
||||
copies_by_layer: "dict[Any, list[_Scatter | None]]" = field(
|
||||
default_factory=lambda: defaultdict(list)
|
||||
)
|
||||
current: "tuple[Any, str] | None" = None
|
||||
# Source names whose ``copy_`` fired during the bake. Names not here never
|
||||
# moved data (e.g. experts owned by another EP rank) and are skipped; names
|
||||
# here but unbaked fail the plan build.
|
||||
copied_names: "set[str]" = field(default_factory=set)
|
||||
|
||||
def accept_copy(self, dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor:
|
||||
self.copied_names.add(src._name)
|
||||
if self.current is not None:
|
||||
layer, param_name = self.current
|
||||
if dest.numel() != src.numel():
|
||||
# The packed layout sizes each slice from the DESTINATION shape,
|
||||
# while the producer packs what the replayed chain produces. A
|
||||
# broadcasting copy_ makes those two lengths differ, and neither
|
||||
# side can notice: the consumer's buffer view is exactly
|
||||
# prod(dest.shape) elements, so it reshapes cleanly over bytes
|
||||
# the producer laid out at different offsets. Refuse at bake.
|
||||
raise _UnsupportedFakeOp(
|
||||
f"FakeRDTTensor: broadcasting copy_ into {param_name!r} of "
|
||||
f"{type(layer).__name__} ({src.numel()} elements from "
|
||||
f"{src._name!r} into {dest.numel()}). The sharded RDT "
|
||||
"backend sends exactly the slice the loader asks for, so "
|
||||
"the source and destination must have equal numel."
|
||||
)
|
||||
self.copies_by_layer[layer].append(
|
||||
_Scatter(
|
||||
layer=layer,
|
||||
param_name=param_name,
|
||||
src=src._key(),
|
||||
offset=dest.storage_offset(),
|
||||
shape=tuple(dest.shape),
|
||||
stride=tuple(dest.stride()),
|
||||
dtype=src.dtype,
|
||||
)
|
||||
)
|
||||
return _meta_copy_(dest, src)
|
||||
|
||||
|
||||
class FakeRDTTensor(torch.Tensor):
|
||||
"""Zero-storage tensor that records how to fetch a weight slice.
|
||||
|
||||
``_make_wrapper_subclass`` gives it shape/dtype/device without storage. Every
|
||||
op in ``SUPPORTED_OPS`` returns a child with the spec appended; ``copy_``
|
||||
delegates to the installed sink. Anything else reaches ``__torch_dispatch__``
|
||||
and raises ``_UnsupportedFakeOp``, so failures are loud rather than silently
|
||||
fetching the wrong bytes.
|
||||
"""
|
||||
|
||||
# Declared at class scope so mypy can infer attribute types on
|
||||
# instances built via ``_make_wrapper_subclass`` (where ``__new__``
|
||||
# returns a tensor it can't annotate as ``self``).
|
||||
_name: str
|
||||
_ops: OpChain
|
||||
# Handles this fake's copy_. ``None`` only on bare construction.
|
||||
_sink: "BakeSink | None"
|
||||
|
||||
@staticmethod
|
||||
def __new__(
|
||||
cls,
|
||||
name: str,
|
||||
shape: torch.Size,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
ops: OpChain = (),
|
||||
sink: "BakeSink | None" = None,
|
||||
) -> "FakeRDTTensor":
|
||||
t = torch.Tensor._make_wrapper_subclass(
|
||||
cls,
|
||||
shape,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
requires_grad=False,
|
||||
)
|
||||
t._name = name
|
||||
t._ops = tuple(ops)
|
||||
t._sink = sink
|
||||
return t
|
||||
|
||||
def _key(self) -> FetchKey:
|
||||
return (self._name, self._ops)
|
||||
|
||||
def _make_child(
|
||||
self,
|
||||
new_shape: torch.Size,
|
||||
new_dtype: torch.dtype,
|
||||
*new_ops: OpSpec,
|
||||
) -> "FakeRDTTensor":
|
||||
"""Append one or more ops to the chain and return a fresh child.
|
||||
|
||||
Variadic so multi-return ops (e.g. chunk) can append both the base
|
||||
op and an indexing op in a single call.
|
||||
"""
|
||||
return FakeRDTTensor(
|
||||
name=self._name,
|
||||
shape=new_shape,
|
||||
dtype=new_dtype,
|
||||
device=self.device,
|
||||
ops=self._ops + new_ops,
|
||||
sink=self._sink,
|
||||
)
|
||||
|
||||
def _meta(self) -> torch.Tensor:
|
||||
"""A zero-storage meta tensor of this fake's shape/dtype, so PyTorch
|
||||
itself computes post-op geometry rather than us reimplementing it."""
|
||||
return torch.empty(self.shape, dtype=self.dtype, device="meta")
|
||||
|
||||
@classmethod
|
||||
def __torch_function__(
|
||||
cls,
|
||||
func,
|
||||
types,
|
||||
args=(),
|
||||
kwargs=None,
|
||||
):
|
||||
kwargs = kwargs or {}
|
||||
|
||||
# Checked before the allowlist: copy_ must not be recorded into the chain.
|
||||
if func is torch.Tensor.copy_:
|
||||
dest = args[0]
|
||||
src = args[1] if len(args) > 1 else kwargs.get("src")
|
||||
if isinstance(src, cls) and src._sink is not None:
|
||||
return src._sink.accept_copy(dest, src)
|
||||
|
||||
# Allowlisted slice/view/shape ops: append to chain and return child.
|
||||
op_name = SUPPORTED_OPS.get(func)
|
||||
if op_name is not None:
|
||||
self_ = args[0]
|
||||
if isinstance(self_, cls):
|
||||
rest = tuple(args[1:])
|
||||
return cls._intercept(self_, func, op_name, rest, kwargs)
|
||||
|
||||
# Metadata reads are answered by the wrapper subclass and stop here; ops
|
||||
# that need data fall through to __torch_dispatch__, which raises.
|
||||
with torch._C.DisableTorchFunctionSubclass():
|
||||
return func(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def _intercept(
|
||||
cls,
|
||||
self_: "FakeRDTTensor",
|
||||
func: Callable,
|
||||
op_name: str,
|
||||
args: tuple,
|
||||
kwargs: dict,
|
||||
):
|
||||
"""Append the op and return a child, or a tuple of children for
|
||||
multi-return ops. Each child's geometry comes from running the op on a
|
||||
meta tensor."""
|
||||
meta = self_._meta()
|
||||
with torch._C.DisableTorchFunctionSubclass():
|
||||
meta_result = func(meta, *args, **kwargs)
|
||||
|
||||
base_op: OpSpec = (op_name, tuple(args), _freeze_kwargs(kwargs))
|
||||
|
||||
# Single-tensor result: one child carrying base_op alone.
|
||||
if isinstance(meta_result, torch.Tensor):
|
||||
return self_._make_child(meta_result.shape, meta_result.dtype, base_op)
|
||||
|
||||
# Multi-return: one child per output, each carrying base_op plus a
|
||||
# trailing __getitem__(i) so the replay can index the result.
|
||||
if isinstance(meta_result, tuple | list):
|
||||
return tuple(
|
||||
self_._make_child(
|
||||
m.shape,
|
||||
m.dtype,
|
||||
base_op,
|
||||
("__getitem__", (i,), ()),
|
||||
)
|
||||
for i, m in enumerate(meta_result)
|
||||
)
|
||||
|
||||
# Non-tensor result (e.g. .item() snuck into the allowlist).
|
||||
raise _UnsupportedFakeOp(
|
||||
f"FakeRDTTensor: {op_name!r} returned a non-tensor "
|
||||
f"({type(meta_result).__name__}); cannot defer."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def __torch_dispatch__(cls, func, types, args=(), kwargs=None):
|
||||
kwargs = kwargs or {}
|
||||
|
||||
# A fake at the aten level means an unsupported loader op. Name the op and
|
||||
# the chain; materializing instead would mask a correctness bug.
|
||||
for arg in (*args, *kwargs.values()):
|
||||
if isinstance(arg, cls):
|
||||
raise _UnsupportedFakeOp(
|
||||
f"FakeRDTTensor: unsupported op {func} reached "
|
||||
f"__torch_dispatch__ on fake {arg._name!r} "
|
||||
f"(chain={arg._ops}). Supported ops are: "
|
||||
f"{sorted(SUPPORTED_OPS.values())}, plus copy_. "
|
||||
"Loaders that need .to(), .float(), .item(), arithmetic, "
|
||||
"bool-mask indexing, or .data access are not supported by "
|
||||
"the sharded RDT backend."
|
||||
)
|
||||
return func(*args, **kwargs)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,377 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sparse NCCL weight transfer engine.
|
||||
|
||||
Sparse patches use checkpoint names, shapes, and flat indices. The model's native
|
||||
weight loader maps them to rank-local runtime parameters, including TP shards and
|
||||
packed parameters.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar, Self
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
TrainerInitInfo,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
WeightTransferEngine,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
worker_init_payload,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
trainer_init as open_trainer_endpoint,
|
||||
)
|
||||
from vllm.model_executor.model_loader.checkpoint_weight_patch import (
|
||||
CheckpointWeightPatch,
|
||||
load_checkpoint_weight_patches,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SparseWeightPatch",
|
||||
"SparseNCCLTrainerInitInfo",
|
||||
"SparseNCCLWeightTransferUpdateInfo",
|
||||
"SparseNCCLWeightTransferEngine",
|
||||
"SparseNCCLTrainerWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseWeightPatch:
|
||||
"""A sparse patch in checkpoint coordinates."""
|
||||
|
||||
name: str
|
||||
indices: torch.Tensor
|
||||
values: torch.Tensor
|
||||
full_shape: tuple[int, ...]
|
||||
"""Full checkpoint shape."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseNCCLTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for the sparse NCCL weight transfer backend.
|
||||
|
||||
Same rendezvous shape as the dense NCCL backend (the sender opens its
|
||||
endpoint as NCCL rank 0), but with no packed wire params: sparse transfers
|
||||
are never packed. `backend` is the factory dispatch key."""
|
||||
|
||||
backend: ClassVar[str] = "sparse_nccl"
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Update info for the sparse NCCL weight transfer backend."""
|
||||
|
||||
names: list[str]
|
||||
dtype_names: list[str]
|
||||
shapes: list[list[int]]
|
||||
num_updates_list: list[int]
|
||||
"""Number of sparse entries to receive for each parameter in ``names``."""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
num_params = len(self.names)
|
||||
if len(self.dtype_names) != num_params:
|
||||
raise ValueError(
|
||||
f"`dtype_names` should be of the same size as `names`: "
|
||||
f"got {len(self.dtype_names)} and {len(self.names)}"
|
||||
)
|
||||
if len(self.shapes) != num_params:
|
||||
raise ValueError(
|
||||
f"`shapes` should be of the same size as `names`: "
|
||||
f"got {len(self.shapes)} and {len(self.names)}"
|
||||
)
|
||||
if len(self.num_updates_list) == 0:
|
||||
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
|
||||
if len(self.num_updates_list) != num_params:
|
||||
raise ValueError(
|
||||
f"`num_updates_list` should be of the same size as `names`: "
|
||||
f"got {len(self.num_updates_list)} and {len(self.names)}"
|
||||
)
|
||||
if any(num_updates < 0 for num_updates in self.num_updates_list):
|
||||
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
|
||||
|
||||
|
||||
class SparseNCCLWeightTransferEngine(
|
||||
WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo]
|
||||
):
|
||||
"""Sparse weight transfer engine using NCCL.
|
||||
|
||||
Receives checkpoint-coordinate patches broadcast from the trainer and applies
|
||||
them through the model's native weight loader. Sparse updates modify initialized
|
||||
model tensors in place, so the layerwise reload lifecycle is not used.
|
||||
"""
|
||||
|
||||
init_info_cls = NCCLWeightTransferInitInfo
|
||||
update_info_cls = SparseNCCLWeightTransferUpdateInfo
|
||||
supports_draft_weight_update = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
|
||||
"""Initialize the NCCL process group with the trainer."""
|
||||
self.model_update_group = worker_init_process_group(
|
||||
init_info, self.parallel_config
|
||||
)
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""No-op: sparse patches are applied in place, no layerwise reload."""
|
||||
pass
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""No-op: sparse patches are applied in place, no layerwise reload."""
|
||||
pass
|
||||
|
||||
def receive_weights(self, update_info: SparseNCCLWeightTransferUpdateInfo) -> None:
|
||||
"""Receive sparse flat-index patches from the trainer and apply them."""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"NCCL weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
|
||||
# Receive on the communicator's device rather than relying on the
|
||||
# ambient current device in the RPC thread.
|
||||
device = self.model_update_group.device
|
||||
with torch.accelerator.device_index(device.index):
|
||||
stream = torch.cuda.current_stream(device=device)
|
||||
patches = []
|
||||
for name, dtype_name, shape, num_updates in zip(
|
||||
update_info.names,
|
||||
update_info.dtype_names,
|
||||
update_info.shapes,
|
||||
update_info.num_updates_list,
|
||||
strict=True,
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
|
||||
values = torch.empty(num_updates, dtype=dtype, device=device)
|
||||
self.model_update_group.broadcast(indices, src=0, stream=stream)
|
||||
self.model_update_group.broadcast(values, src=0, stream=stream)
|
||||
patches.append(
|
||||
CheckpointWeightPatch(
|
||||
name=name,
|
||||
shape=tuple(shape),
|
||||
dtype=dtype,
|
||||
values=values,
|
||||
indices=indices,
|
||||
)
|
||||
)
|
||||
load_checkpoint_weight_patches(self.model, patches)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
|
||||
|
||||
class SparseNCCLTrainerWeightTransferEngine(
|
||||
TrainerWeightTransferEngine[SparseNCCLTrainerInitInfo]
|
||||
):
|
||||
"""Trainer-side sparse NCCL weight transfer engine.
|
||||
|
||||
Broadcasts flat-index (indices, values) patches from NCCL rank 0 while the
|
||||
inference-side `update_weights` runs concurrently on a side thread (the
|
||||
worker's recvs rendezvous inside the same NCCL broadcasts). `send_weights`
|
||||
owns a complete one-shot lifecycle. RL infrastructure that owns the generic
|
||||
client lifecycle can call `send_weight_chunk` between one start and finish.
|
||||
|
||||
Sparse patches differ every round, so they are not a stable `WeightSource`:
|
||||
the engine takes no `source`, and patches are passed directly to the send
|
||||
methods. An empty patch list is a no-op.
|
||||
|
||||
Only the designated trainer sender joins the transfer group; other trainer
|
||||
ranks skip sparse sends.
|
||||
"""
|
||||
|
||||
init_info_cls = SparseNCCLTrainerInitInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
is_sender: bool = True,
|
||||
) -> None:
|
||||
# Sparse is a delta backend: it takes per-round patches via
|
||||
# send_weights, so a `source` would silently never be sent. The
|
||||
# parameter exists only to match the base/factory signature.
|
||||
if source is not None:
|
||||
raise ValueError(
|
||||
"Sparse NCCL weight transfer takes no WeightSource; pass each "
|
||||
"round's patches to send_weights(patches) instead."
|
||||
)
|
||||
super().__init__(client=client, source=source, is_sender=is_sender)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
init_info: SparseNCCLTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
engine = cls(client=client, source=source, is_sender=init_info.is_sender)
|
||||
if not engine.is_sender:
|
||||
return engine
|
||||
|
||||
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
|
||||
# Sparse transfers are never packed, so the worker keeps the unpacked
|
||||
# defaults on its init info.
|
||||
worker_init_info = NCCLWeightTransferInitInfo(
|
||||
master_address=init_info.master_address,
|
||||
master_port=init_info.master_port,
|
||||
rank_offset=1,
|
||||
world_size=init_info.world_size,
|
||||
)
|
||||
|
||||
# The inference workers block inside init_weight_transfer_engine waiting
|
||||
# for the NCCL rendezvous, so we kick that off on a side thread while we
|
||||
# open the trainer endpoint (rank 0); both sides must rendezvous together.
|
||||
with ThreadPoolExecutor(max_workers=1) as exe:
|
||||
future = exe.submit(
|
||||
engine.client.init_weight_transfer_engine,
|
||||
worker_init_payload(worker_init_info),
|
||||
)
|
||||
# Open the trainer endpoint as NCCL rank 0 on the current device
|
||||
# (the init info satisfies the helper's rendezvous protocol).
|
||||
engine.model_update_group = open_trainer_endpoint(init_info)
|
||||
future.result() # surface any inference-side init error
|
||||
|
||||
return engine
|
||||
|
||||
def send_weights(self, patches: Iterable[SparseWeightPatch] | None = None) -> None:
|
||||
"""Broadcast one sparse update through a one-shot lifecycle."""
|
||||
patches = self._prepare_patches(patches)
|
||||
if not patches:
|
||||
return
|
||||
|
||||
self.client.start_weight_update()
|
||||
self._broadcast_chunk(patches)
|
||||
self.client.finish_weight_update()
|
||||
|
||||
def send_weight_chunk(
|
||||
self, patches: Iterable[SparseWeightPatch] | None = None
|
||||
) -> None:
|
||||
"""Broadcast one chunk inside a caller-owned weight update lifecycle."""
|
||||
patches = self._prepare_patches(patches)
|
||||
if not patches:
|
||||
return
|
||||
self._broadcast_chunk(patches)
|
||||
|
||||
def _prepare_patches(
|
||||
self, patches: Iterable[SparseWeightPatch] | None
|
||||
) -> list[SparseWeightPatch]:
|
||||
if not self.is_sender:
|
||||
return []
|
||||
|
||||
prepared = list(patches) if patches is not None else []
|
||||
if not prepared:
|
||||
return []
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError("trainer_init() must be called before sending weights.")
|
||||
for patch in prepared:
|
||||
self._validate_patch(patch)
|
||||
return prepared
|
||||
|
||||
def _broadcast_chunk(self, patches: list[SparseWeightPatch]) -> None:
|
||||
assert self.model_update_group is not None
|
||||
device = self.model_update_group.device
|
||||
device_context = (
|
||||
torch.accelerator.device_index(device.index)
|
||||
if device.type != "cpu"
|
||||
else nullcontext()
|
||||
)
|
||||
with device_context:
|
||||
update_info = SparseNCCLWeightTransferUpdateInfo(
|
||||
names=[patch.name for patch in patches],
|
||||
dtype_names=[
|
||||
str(patch.values.dtype).split(".")[-1] for patch in patches
|
||||
],
|
||||
shapes=[list(patch.full_shape) for patch in patches],
|
||||
num_updates_list=[patch.indices.numel() for patch in patches],
|
||||
)
|
||||
|
||||
# update_weights (workers receive) must run concurrently with the
|
||||
# trainer-side broadcasts — both rendezvous inside the same NCCL calls.
|
||||
executor = ThreadPoolExecutor(max_workers=1)
|
||||
try:
|
||||
future = executor.submit(
|
||||
self.client.update_weights, asdict(update_info)
|
||||
)
|
||||
# Surface an RPC that failed before the worker entered NCCL rather
|
||||
# than broadcasting to a peer that will never receive.
|
||||
if future.done():
|
||||
future.result()
|
||||
stream = torch.cuda.current_stream()
|
||||
for patch in patches:
|
||||
self.model_update_group.broadcast(
|
||||
patch.indices, src=0, stream=stream
|
||||
)
|
||||
self.model_update_group.broadcast(
|
||||
patch.values, src=0, stream=stream
|
||||
)
|
||||
future.result() # surface inference-side errors
|
||||
finally:
|
||||
# A failed broadcast can leave the RPC blocked in the matching
|
||||
# receive. Waiting for that thread would hide the original error.
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
self._post_send_sync()
|
||||
|
||||
@staticmethod
|
||||
def _validate_patch(patch: SparseWeightPatch) -> None:
|
||||
"""Reject a malformed patch before starting the NCCL transfer."""
|
||||
if patch.full_shape is None:
|
||||
raise ValueError(f"Sparse patch requires full_shape: {patch.name}")
|
||||
if patch.indices.dtype != torch.int32:
|
||||
raise ValueError(
|
||||
f"Sparse weight updates require int32 indices: {patch.name}"
|
||||
)
|
||||
if patch.indices.ndim != 1 or patch.values.ndim != 1:
|
||||
raise ValueError(
|
||||
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
|
||||
)
|
||||
if patch.indices.numel() != patch.values.numel():
|
||||
raise ValueError(
|
||||
f"`indices` and `values` must have matching lengths for {patch.name}"
|
||||
)
|
||||
|
||||
def _post_send_sync(self) -> None:
|
||||
"""Wait for the broadcasts to land before returning, so a caller may
|
||||
rebuild or free the patch tensors as soon as a send method returns rather
|
||||
than relying on same-stream ordering. See
|
||||
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
|
||||
cross-rank barrier."""
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group.destroy()
|
||||
self.model_update_group = None
|
||||
Reference in New Issue
Block a user