[rl] Stateful Trainer Send: NCCL + Sparse NCCL [3/N] (#50902)

Signed-off-by: haoaaron <ahao@anyscale.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Aaron Hao
2026-08-07 00:35:59 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 43d691ec6b
commit 21ea5b4fa1
14 changed files with 1422 additions and 783 deletions
+10 -10
View File
@@ -77,6 +77,7 @@ class WeightSource(ABC):
@abstractmethod
def metadata(self) -> list[ParamMeta]:
"""Declare what iteration will yield, without transferring anything."""
raise NotImplementedError
@abstractmethod
@@ -402,8 +403,7 @@ class TrainerWeightTransferEngine(ABC, Generic[TTrainerInitInfo]):
Symmetric to `WeightTransferEngine` but lives in the training process.
Constructed via the `trainer_init` factory classmethod; carries any
backend-specific state (NCCL communicators, IPC device info, transfer
plans) on `self`. The `WeightSource` is required at `trainer_init`,
then replayed each round by the no-argument `send_weights()`.
plans) on `self`.
Unlike the worker engine, the trainer side does not take a
`WeightTransferConfig`: the backend is selected from the init info's
@@ -431,7 +431,7 @@ class TrainerWeightTransferEngine(ABC, Generic[TTrainerInitInfo]):
self,
*,
client: "VLLMWeightSyncClient",
source: "WeightSource",
source: "WeightSource | None" = None,
is_sender: bool = True,
) -> None:
self.is_sender = is_sender
@@ -447,24 +447,24 @@ class TrainerWeightTransferEngine(ABC, Generic[TTrainerInitInfo]):
init_info: TTrainerInitInfo,
*,
client: "VLLMWeightSyncClient",
source: "WeightSource",
source: "WeightSource | None" = None,
) -> Self:
"""Rendezvous with the inference side and return a ready instance.
Called on every trainer rank. The sender drives the full handshake via
`client` (build the worker-side init info, call
`client.init_weight_transfer_engine`, open the trainer-side endpoint);
non-sender ranks skip the rendezvous and the RPC. `source` is stored on
`self.source`; after return, `send_weights()` is callable.
non-sender ranks skip the rendezvous and the RPC.
"""
raise NotImplementedError
@abstractmethod
def send_weights(self) -> None:
"""Push `self.source`'s weights to inference workers and drive the full
update round trip: `start_weight_update`, `update_weights` (run
concurrently with the trainer-side broadcast when the backend requires
it), then `finish_weight_update`. Called on every trainer rank."""
"""Push weights to inference workers and drive the full update round
trip: `start_weight_update`, `update_weights` (run concurrently with the
trainer-side broadcast when the backend requires it), then
`finish_weight_update`. Called on every trainer rank.
"""
raise NotImplementedError
def shutdown(self) -> None:
+17 -4
View File
@@ -167,7 +167,7 @@ class WeightTransferTrainerFactory:
init_info: "TrainerInitInfo",
*,
client: "VLLMWeightSyncClient",
source: "WeightSource",
source: "WeightSource | None" = None,
) -> TrainerWeightTransferEngine:
"""Build and rendezvous a ready-to-send trainer engine.
@@ -184,7 +184,9 @@ class WeightTransferTrainerFactory:
selects the engine; it also carries the wire params (e.g.
`packed`).
client: Inference-side control-plane client.
source: `WeightSource` of `(name, tensor)` pairs to send each round.
source: `WeightSource` of `(name, tensor)` pairs to send each round,
for full-resync backends (NCCL, IPC). Sparse backend
omits it and passes its per-round payload to `send_weights`.
Raises:
ValueError: If `init_info.backend` is not registered.
@@ -233,10 +235,21 @@ WeightTransferEngineFactory.register_engine(
)
# Trainer-side engines. Backends register here as they migrate to the stateful
# trainer engine; NCCL / sparse NCCL keep their static trainer path until then.
# Trainer-side engines, parallel to the worker registry above.
WeightTransferTrainerFactory.register_engine(
"nccl",
"vllm.distributed.weight_transfer.nccl_engine",
"NCCLTrainerWeightTransferEngine",
)
WeightTransferTrainerFactory.register_engine(
"ipc",
"vllm.distributed.weight_transfer.ipc_engine",
"IPCTrainerWeightTransferEngine",
)
WeightTransferTrainerFactory.register_engine(
"sparse_nccl",
"vllm.distributed.weight_transfer.sparse_nccl_engine",
"SparseNCCLTrainerWeightTransferEngine",
)
@@ -291,8 +291,10 @@ class IPCTrainerWeightTransferEngine(TrainerWeightTransferEngine[IPCTrainerInitI
init_info: IPCTrainerInitInfo,
*,
client: VLLMWeightSyncClient,
source: WeightSource,
source: WeightSource | None = None,
) -> Self:
if source is None:
raise ValueError("IPC trainer weight transfer requires a WeightSource.")
engine = cls(
client=client,
source=source,
@@ -307,6 +309,7 @@ class IPCTrainerWeightTransferEngine(TrainerWeightTransferEngine[IPCTrainerInitI
return engine
def send_weights(self) -> None:
assert self.source is not None
source = self.source
if self.is_sender:
self.client.start_weight_update()
@@ -9,7 +9,7 @@ sparse engine does not have to subclass the dense one.
"""
from dataclasses import dataclass
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Protocol
import torch
@@ -18,16 +18,36 @@ if TYPE_CHECKING:
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
DEFAULT_PACKED_NUM_BUFFERS,
)
@dataclass
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
"""Initialization info for NCCL-based weight transfer backends."""
"""Worker-side initialization info for NCCL-based weight transfer backends."""
master_address: str
master_port: int
rank_offset: int
world_size: int
packed: bool = False
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
class NCCLRendezvous(Protocol):
"""The three fields `trainer_init` needs to open the trainer endpoint.
Structural so each backend can keep its own trainer init info next to its
engine (dense `NCCLTrainerInitInfo`, `SparseNCCLTrainerInitInfo`) without
this shared module importing either.
"""
master_address: str
master_port: int
world_size: int
def stateless_init_process_group(
@@ -83,7 +103,7 @@ def worker_init_process_group(
def trainer_init(
init_info: NCCLWeightTransferInitInfo | dict,
init_info: NCCLRendezvous | dict,
) -> "PyNcclCommunicator":
"""
Initialize NCCL process group for trainer-side weight transfer.
@@ -92,7 +112,8 @@ def trainer_init(
CUDA device (torch.accelerator.current_device_index()).
Args:
init_info: Either an NCCLWeightTransferInitInfo object or a dict with keys:
init_info: Any object carrying the `NCCLRendezvous` fields (a trainer or
worker NCCL init info), or a dict with keys:
- master_address: str
- master_port: int
- world_size: int
+292 -97
View File
@@ -2,11 +2,13 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NCCL-based (dense) weight transfer engine."""
from collections.abc import Callable, Iterator
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from dataclasses import asdict, dataclass
from typing import TYPE_CHECKING, Any, ClassVar
import torch
from typing_extensions import Self
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -14,72 +16,76 @@ if TYPE_CHECKING:
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
ParamMeta,
TrainerInitInfo,
TrainerWeightTransferEngine,
VLLMWeightSyncClient,
WeightSource,
WeightTransferEngine,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
trainer_init,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_common import (
trainer_init as open_trainer_endpoint,
)
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
DEFAULT_PACKED_NUM_BUFFERS,
packed_nccl_broadcast_consumer,
packed_nccl_broadcast_producer,
)
# Re-exported for backward compatibility; canonical home is nccl_common.
# NCCLWeightTransferInitInfo is re-exported here for convenience; its canonical
# home is nccl_common, shared with the sparse backend.
__all__ = [
"NCCLWeightTransferInitInfo",
"NCCLTrainerSendWeightsArgs",
"NCCLTrainerInitInfo",
"NCCLWeightTransferUpdateInfo",
"NCCLWeightTransferEngine",
"NCCLTrainerWeightTransferEngine",
]
@dataclass
class NCCLTrainerSendWeightsArgs:
"""Arguments for NCCL trainer_send_weights method."""
class NCCLTrainerInitInfo(TrainerInitInfo):
"""Trainer-side init info for the dense NCCL weight transfer backend.
group: Any
"""Process group (PyNcclCommunicator) for NCCL communication."""
src: int = 0
"""Source rank (default 0, trainer is typically rank 0)."""
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor] | None = None
"""Optional function to apply to each (name, tensor) pair before broadcasting.
If None, extracts just the tensor."""
packed: bool = False
"""Whether to use packed tensor broadcasting for efficiency.
When True, multiple tensors are batched together before broadcasting
to reduce NCCL communication overhead."""
stream: torch.cuda.Stream | None = None
"""CUDA stream to use for broadcasting if packed is False.
If packed is True, new streams will be created for each buffer."""
The sender opens its endpoint as NCCL rank 0, so it needs no `rank_offset`.
`world_size` is the full trainer+worker NCCL group size. `rank` (from
`TrainerInitInfo`) identifies this trainer process; rank 0 is the sender.
`packed` / buffer sizes are the transfer's wire params. The trainer
propagates them to the worker at `trainer_init` so the two sides cannot
disagree. Note this defaults to packed, unlike the worker-side
`NCCLWeightTransferInitInfo`, whose default only applies when no trainer
ships a value. `backend` is the factory dispatch key."""
backend: ClassVar[str] = "nccl"
master_address: str
master_port: int
world_size: int
packed: bool = True
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
"""Size in bytes for each packed tensor buffer.
Must match the value used in NCCLWeightTransferUpdateInfo."""
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
"""Number of buffers for double/triple buffering during packed transfer.
Must match the value used in NCCLWeightTransferUpdateInfo."""
@dataclass
class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Update info for NCCL weight transfer backend."""
"""Per-round update info for the dense NCCL weight transfer backend.
Whether the transfer is packed (and the buffer geometry) is a must-agree
wire param carried on the init info (`NCCLTrainerInitInfo` /
`NCCLWeightTransferInitInfo`), not here; this carries only the per-round
parameter metadata.
"""
names: list[str]
dtype_names: list[str]
shapes: list[list[int]]
packed: bool = False
"""Whether to use packed tensor broadcasting for efficiency.
When True, multiple tensors are batched together before broadcasting
to reduce NCCL communication overhead."""
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
"""Size in bytes for each packed tensor buffer.
Both producer and consumer must use the same value."""
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
"""Number of buffers for double/triple buffering during packed transfer.
Both producer and consumer must use the same value."""
def __post_init__(self):
"""Validate that all lists have the same length."""
@@ -121,15 +127,25 @@ class NCCLWeightTransferEngine(
) -> None:
super().__init__(config, vllm_config, device, model)
self.model_update_group: PyNcclCommunicator | None = None
# Set from the trainer-supplied init info at the handshake; defaults are
# only for the (unreachable) receive-before-init case.
self.packed = False
self.packed_buffer_size_bytes = DEFAULT_PACKED_BUFFER_SIZE_BYTES
self.packed_num_buffers = DEFAULT_PACKED_NUM_BUFFERS
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
"""
Initialize NCCL process group with the trainer.
Initialize NCCL process group with the trainer and record the
trainer-supplied wire params so the worker decodes exactly as the
trainer encodes.
Args:
init_info: NCCL initialization info containing master address, port,
rank offset, and world size
rank offset, world size, and the packed wire params
"""
self.packed = init_info.packed
self.packed_buffer_size_bytes = init_info.packed_buffer_size_bytes
self.packed_num_buffers = init_info.packed_num_buffers
self.model_update_group = worker_init_process_group(
init_info, self.parallel_config
)
@@ -154,13 +170,14 @@ class NCCLWeightTransferEngine(
"""
Receive weights from trainer via NCCL broadcast.
If update_info.packed is True, uses packed tensor broadcasting for
efficient transfer of multiple weights in batches. Otherwise, uses simple
one-by-one broadcasting.
Whether to use packed broadcasting (and the buffer geometry) is read
from `self.packed` / `self.packed_*`, set at the init handshake from the
trainer's init info, so it is guaranteed to match how the trainer
encoded.
Args:
update_info: NCCL update info containing parameter names, dtypes, shapes,
and packed flag
update_info: NCCL update info containing parameter names, dtypes,
and shapes
"""
if self.model_update_group is None:
raise RuntimeError(
@@ -173,7 +190,7 @@ class NCCLWeightTransferEngine(
)
with disable_mtp_completeness_check():
if update_info.packed:
if self.packed:
# Build iterator of (name, (shape, dtype)) from update_info
def state_dict_info_iterator():
for name, dtype_name, shape in zip(
@@ -187,8 +204,8 @@ class NCCLWeightTransferEngine(
group=self.model_update_group,
src=0,
post_unpack_func=self.model.load_weights,
buffer_size_bytes=update_info.packed_buffer_size_bytes,
num_buffers=update_info.packed_num_buffers,
buffer_size_bytes=self.packed_buffer_size_bytes,
num_buffers=self.packed_num_buffers,
device=self.device,
)
else:
@@ -210,63 +227,241 @@ class NCCLWeightTransferEngine(
self.model_update_group = None
@staticmethod
def trainer_send_weights(
iterator: Iterator[tuple[str, torch.Tensor]],
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
) -> None:
"""Broadcast dense weights from trainer to vLLM workers.
def trainer_send_weights(*args: Any, **kwargs: Any) -> None:
"""Removed. Use the stateful `NCCLTrainerWeightTransferEngine` instead.
Args:
iterator: Iterator of model parameters. Returns (name, tensor) tuples
trainer_args: Dictionary or NCCLTrainerSendWeightsArgs instance containing
NCCL-specific arguments. If a dict, should contain keys from
NCCLTrainerSendWeightsArgs.
Example:
>>> from vllm.distributed.weight_transfer.nccl_engine import (
... NCCLWeightTransferEngine,
... NCCLTrainerSendWeightsArgs,
... )
>>> param_iter = ((n, p) for n, p in model.named_parameters())
>>> args = NCCLTrainerSendWeightsArgs(group=group, packed=True)
>>> NCCLWeightTransferEngine.trainer_send_weights(param_iter, args)
Transitional stub kept only to satisfy the (still abstract)
`WeightTransferEngine.trainer_send_weights`; that member is dropped from
the worker ABC once every backend has migrated to the trainer engine.
"""
# Parse trainer args - accept either dict or dataclass instance
if isinstance(trainer_args, dict):
args = NCCLTrainerSendWeightsArgs(**trainer_args)
else:
args = trainer_args
raise NotImplementedError(
"The static NCCL trainer path has been replaced by "
"NCCLTrainerWeightTransferEngine. Build it via "
"WeightTransferTrainerFactory.trainer_init(NCCLTrainerInitInfo(...), "
"client=..., source=...) and drive it with send_weights()."
)
if args.post_iter_func is None:
# Default: extract just the tensor from (name, tensor) tuple
post_iter_func = lambda x: x[1]
else:
post_iter_func = args.post_iter_func
if args.packed:
# Use packed tensor broadcasting for efficiency
from vllm.distributed.weight_transfer.packed_tensor import (
packed_nccl_broadcast_producer,
class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerInitInfo]):
"""Trainer-side NCCL weight transfer engine.
On the sender (rank 0) holds the NCCL communicator and drives the full
update round trip: it runs the inference-side `update_weights` concurrently
with the trainer-side broadcast (both rendezvous inside the same NCCL
calls), then finishes the update. Non-sender trainer ranks hold no
communicator; they only iterate the source to stay in the trainer-side
collective (e.g. FSDP `full_tensor()`) and skip the client RPCs and the
broadcast (all guarded on `is_sender`).
`packed` / buffer sizes come from `NCCLTrainerInitInfo`; the sender
propagates them to the worker at `trainer_init` (on the worker-side init
info), so per-round payloads carry only parameter metadata.
"""
init_info_cls = NCCLTrainerInitInfo
def __init__(
self,
*,
client: VLLMWeightSyncClient,
source: WeightSource,
is_sender: bool = True,
packed: bool = True,
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
) -> None:
super().__init__(client=client, source=source, is_sender=is_sender)
self.packed = packed
self.packed_buffer_size_bytes = packed_buffer_size_bytes
self.packed_num_buffers = packed_num_buffers
self.model_update_group: PyNcclCommunicator | None = None
@classmethod
def trainer_init(
cls,
init_info: NCCLTrainerInitInfo,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
) -> Self:
if source is None:
raise ValueError("NCCL trainer weight transfer requires a WeightSource.")
engine = cls(
client=client,
source=source,
is_sender=init_info.is_sender,
packed=init_info.packed,
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
packed_num_buffers=init_info.packed_num_buffers,
)
if not engine.is_sender:
# Non-sender trainer ranks aren't part of the transfer NCCL group and
# don't drive the inference side; they only participate in the
# trainer-side gather during send_weights.
return engine
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
worker_init_info = NCCLWeightTransferInitInfo(
master_address=init_info.master_address,
master_port=init_info.master_port,
rank_offset=1,
world_size=init_info.world_size,
packed=init_info.packed,
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
packed_num_buffers=init_info.packed_num_buffers,
)
# The inference workers block inside init_weight_transfer_engine waiting
# for the NCCL rendezvous, so we kick that off on a side thread while we
# open the trainer endpoint (rank 0); both sides must rendezvous together.
with ThreadPoolExecutor(max_workers=1) as exe:
future = exe.submit(
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
)
engine.model_update_group = open_trainer_endpoint(init_info)
future.result() # surface any inference-side init error
return engine
def send_weights(self) -> None:
assert self.source is not None # guaranteed by trainer_init / __init__
source = self.source
# Metadata is declared without gathering. For Megatron it is itself a
# collective, so every rank runs it; only the sender ships it.
meta = source.metadata()
if not self.is_sender:
# Non-sender ranks only join the trainer-side gather collective.
self._broadcast(source, meta)
self._post_send_sync()
return
update_info = NCCLWeightTransferUpdateInfo(
names=[m.name for m in meta],
dtype_names=[str(m.dtype).split(".")[-1] for m in meta],
shapes=[list(m.shape) for m in meta],
)
self.client.start_weight_update()
# update_weights (workers receive) must run concurrently with the
# trainer-side broadcast — both rendezvous inside the same NCCL calls.
exe = ThreadPoolExecutor(max_workers=1)
try:
future = exe.submit(self.client.update_weights, asdict(update_info))
# Cheap best-effort: if update_weights already failed (e.g. a bad
# request rejected before any NCCL call), surface it now instead of
# hanging in broadcast waiting for a peer that will never arrive.
if future.done():
future.result()
self._broadcast(source, meta)
future.result() # surface inference-side errors
finally:
# Never wait for the RPC thread here. If the broadcast raised, the
# worker is still blocked in the matching NCCL call and will never
# return, so joining would turn the error into a permanent hang.
# Let the exception out instead; the transfer group is unusable
# either way and the caller has to tear it down. (The orphaned
# thread is still joined at interpreter exit, so a caller that wants
# to exit cleanly after such a failure must not wait on it.)
exe.shutdown(wait=False)
self.client.finish_weight_update()
self._post_send_sync()
def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None:
"""Iterate the source (materializing each tensor — a collective on all
ranks) and, on the sender, broadcast from rank 0, packed or one-by-one.
Non-sender ranks only replay the iteration to stay in the collective."""
if not self.is_sender:
for _ in source:
pass
return
assert self.model_update_group is not None, (
"trainer_init() must be called before _broadcast()."
)
pairs = self._checked_iter(source, meta)
if self.packed:
packed_nccl_broadcast_producer(
iterator=iterator,
group=args.group,
src=args.src,
post_iter_func=post_iter_func,
buffer_size_bytes=args.packed_buffer_size_bytes,
num_buffers=args.packed_num_buffers,
iterator=pairs,
group=self.model_update_group,
src=0,
post_iter_func=lambda item: item[1],
buffer_size_bytes=self.packed_buffer_size_bytes,
num_buffers=self.packed_num_buffers,
)
else:
# Use simple one-by-one broadcasting
for item in iterator:
tensor = post_iter_func(item)
args.group.broadcast(
tensor,
src=args.src,
stream=args.stream or torch.cuda.current_stream(),
)
stream = torch.cuda.current_stream()
for _name, tensor in pairs:
# NCCL sends `numel` elements straight from `data_ptr()`, so a
# non-contiguous view would ship whatever follows its base
# pointer. Keep the copy referenced until the broadcast is
# enqueued. (The packed path linearizes in `pack_tensors`.)
send = tensor if tensor.is_contiguous() else tensor.contiguous()
self.model_update_group.broadcast(send, src=0, stream=stream)
# Trainer-side process-group setup. Delegates to the shared helper so the
# sparse engine can reuse the exact same rendezvous without subclassing.
trainer_init = staticmethod(trainer_init)
@staticmethod
def _checked_iter(
source: WeightSource, meta: list[ParamMeta]
) -> Iterator[tuple[str, torch.Tensor]]:
"""Yield the source's pairs, checking each against what the worker was
told to expect.
The worker sizes its receive buffers — and in packed mode cuts its chunk
boundaries — from the update info, which is built from `metadata()`. If
iteration disagrees with it, the two sides split the stream differently
and the transfer hangs in NCCL or loads garbage. Checking here costs one
comparison per parameter and turns that into an error naming the first
divergent parameter. Sender-only: under pipeline parallelism a
non-sender's yielded tensor is not meaningful.
"""
sent = 0
for name, tensor in source:
if sent >= len(meta):
raise ValueError(
f"WeightSource yielded more parameters than metadata() "
f"declared ({len(meta)}); first extra is {name!r}."
)
expected = meta[sent]
if (
name != expected.name
or tensor.dtype != expected.dtype
or tuple(tensor.shape) != expected.shape
):
raise ValueError(
"WeightSource metadata() disagrees with iteration at index "
f"{sent}: declared {expected.name!r} "
f"{expected.dtype} {tuple(expected.shape)}, got {name!r} "
f"{tensor.dtype} {tuple(tensor.shape)}. Both channels must "
"enumerate the same parameters in the same order."
)
sent += 1
yield name, tensor
if sent != len(meta):
raise ValueError(
f"WeightSource yielded {sent} parameters but metadata() "
f"declared {len(meta)}; the worker is waiting for the rest."
)
def _post_send_sync(self) -> None:
"""Wait for this rank's transfer work to land before returning.
Broadcasts are only *enqueued* by `send_weights`: the unpacked path on
the current stream, the packed path on the producer's own streams (which
it drains itself). Waiting here lets a caller mutate parameters, or
start the next step on another stream, as soon as `send_weights`
returns, instead of silently depending on same-stream ordering. Every
rank waits: a non-sender's `full_tensor()` gathers feed the sender's
broadcast, so they must have landed before it may touch its shards.
Unlike IPC there is no cross-rank barrier here. Nothing in this backend
outlives the collective it travelled in (IPC's barrier keeps shared
buffers alive until every consumer has opened them), so a barrier would
only add a dependency on the default process group that this backend
otherwise does not have.
"""
if torch.cuda.is_available():
torch.cuda.current_stream().synchronize()
def shutdown(self) -> None:
self.model_update_group = None
@@ -177,6 +177,10 @@ def packed_nccl_broadcast_producer(
# Move to the next buffer
buffer_idx = (buffer_idx + 1) % num_buffers
# Drain every slot before returning.
for stream in streams:
stream.synchronize()
def packed_nccl_broadcast_consumer(
iterator: Iterator[tuple[str, tuple[list[int], torch.dtype]]],
@@ -279,6 +283,10 @@ def packed_nccl_broadcast_consumer(
)
break
# Drain every slot before returning.
for stream in streams:
stream.synchronize()
# ── IPC packed transfer ────────────────────────────────────────────────
@@ -14,11 +14,13 @@ MVP limitations:
* not composable with checkpoint-format or packed updates
"""
from collections.abc import Iterator
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from collections.abc import Iterable
from concurrent.futures import ThreadPoolExecutor
from dataclasses import asdict, dataclass
from typing import TYPE_CHECKING, Any, ClassVar
import torch
from typing_extensions import Self
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -26,20 +28,27 @@ if TYPE_CHECKING:
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
TrainerInitInfo,
TrainerWeightTransferEngine,
VLLMWeightSyncClient,
WeightSource,
WeightTransferEngine,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
trainer_init,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerSendWeightsArgs
from vllm.distributed.weight_transfer.nccl_common import (
trainer_init as open_trainer_endpoint,
)
__all__ = [
"SparseWeightPatch",
"SparseNCCLTrainerInitInfo",
"SparseNCCLWeightTransferUpdateInfo",
"SparseNCCLWeightTransferEngine",
"SparseNCCLTrainerWeightTransferEngine",
]
@@ -50,6 +59,25 @@ class SparseWeightPatch:
name: str
indices: torch.Tensor
values: torch.Tensor
full_shape: tuple[int, ...] | None = None
"""Full shape of the patched parameter. Required when the patch is sent
via `SparseNCCLTrainerWeightTransferEngine` (it ships in the per-round
update info); the worker-side apply path does not read it."""
@dataclass
class SparseNCCLTrainerInitInfo(TrainerInitInfo):
"""Trainer-side init info for the sparse NCCL weight transfer backend.
Same rendezvous shape as the dense NCCL backend (the sender opens its
endpoint as NCCL rank 0), but with no packed wire params: sparse transfers
are never packed. `backend` is the factory dispatch key."""
backend: ClassVar[str] = "sparse_nccl"
master_address: str
master_port: int
world_size: int
@dataclass
@@ -200,25 +228,182 @@ class SparseNCCLWeightTransferEngine(
self.model_update_group = None
@staticmethod
def trainer_send_weights(
iterator: Iterator[SparseWeightPatch],
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
) -> None:
"""Broadcast sparse flat-index patches from trainer to vLLM workers."""
if isinstance(trainer_args, dict):
args = NCCLTrainerSendWeightsArgs(**trainer_args)
else:
args = trainer_args
def trainer_send_weights(*args: Any, **kwargs: Any) -> None:
"""Removed. Use the stateful `SparseNCCLTrainerWeightTransferEngine`.
if args.packed:
Transitional stub kept only to satisfy the (still abstract)
`WeightTransferEngine.trainer_send_weights`; that member is dropped from
the worker ABC once every backend has migrated to the trainer engine.
"""
raise NotImplementedError(
"The static sparse NCCL trainer path has been replaced by "
"SparseNCCLTrainerWeightTransferEngine. Build it via "
"WeightTransferTrainerFactory.trainer_init("
"SparseNCCLTrainerInitInfo(...), client=..., source=...) and drive "
"it with send_weights()."
)
class SparseNCCLTrainerWeightTransferEngine(
TrainerWeightTransferEngine[SparseNCCLTrainerInitInfo]
):
"""Trainer-side sparse NCCL weight transfer engine.
Broadcasts flat-index (indices, values) patches from NCCL rank 0 while the
inference-side `update_weights` runs concurrently on a side thread (the
worker's recvs rendezvous inside the same NCCL broadcasts), then finishes
the update. Unlike the full-resync backends, sparse patches differ every
round (a fresh set of deltas from each optimizer step), so they are not a
stable `WeightSource`: the engine takes no `source`, and each round's
patches are passed straight to `send_weights(patches)`. A round with no
patches is a no-op.
The sparse backend assumes a single-rank trainer (matching its TP=1 / PP=1
MVP scope), so non-sender ranks skip `send_weights` entirely.
"""
init_info_cls = SparseNCCLTrainerInitInfo
def __init__(
self,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
is_sender: bool = True,
) -> None:
# Sparse is a delta backend: it takes per-round patches via
# send_weights, so a `source` would silently never be sent. The
# parameter exists only to match the base/factory signature.
if source is not None:
raise ValueError(
"Sparse NCCL updates cannot be combined with `packed=True`"
"Sparse NCCL weight transfer takes no WeightSource; pass each "
"round's patches to send_weights(patches) instead."
)
super().__init__(client=client, source=source, is_sender=is_sender)
self.model_update_group: PyNcclCommunicator | None = None
@classmethod
def trainer_init(
cls,
init_info: SparseNCCLTrainerInitInfo,
*,
client: VLLMWeightSyncClient,
source: WeightSource | None = None,
) -> Self:
engine = cls(client=client, source=source, is_sender=init_info.is_sender)
if not engine.is_sender:
return engine
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
# Sparse transfers are never packed, so the worker keeps the unpacked
# defaults on its init info.
worker_init_info = NCCLWeightTransferInitInfo(
master_address=init_info.master_address,
master_port=init_info.master_port,
rank_offset=1,
world_size=init_info.world_size,
)
# The inference workers block inside init_weight_transfer_engine waiting
# for the NCCL rendezvous, so we kick that off on a side thread while we
# open the trainer endpoint (rank 0); both sides must rendezvous together.
with ThreadPoolExecutor(max_workers=1) as exe:
future = exe.submit(
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
)
# Open the trainer endpoint as NCCL rank 0 on the current device
# (the init info satisfies the helper's rendezvous protocol).
engine.model_update_group = open_trainer_endpoint(init_info)
future.result() # surface any inference-side init error
return engine
def send_weights(self, patches: Iterable[SparseWeightPatch] | None = None) -> None:
"""Broadcast this round's sparse patches. `patches` is the per-round
payload (sparse deltas differ every round), so it is passed here rather
than fixed at init. Every patch must set `full_shape`."""
if not self.is_sender:
return
patches = list(patches) if patches is not None else []
if not patches:
return
shapes = []
for patch in patches:
if patch.full_shape is None:
raise ValueError(
"SparseWeightPatch.full_shape must be set to send via the "
f"trainer engine: {patch.name}"
)
self._validate_patch(patch)
shapes.append(list(patch.full_shape))
update_info = SparseNCCLWeightTransferUpdateInfo(
names=[patch.name for patch in patches],
dtype_names=[str(patch.values.dtype).split(".")[-1] for patch in patches],
shapes=shapes,
num_updates_list=[patch.indices.numel() for patch in patches],
)
assert self.model_update_group is not None, (
"trainer_init() must be called before send_weights()."
)
self.client.start_weight_update()
# update_weights (workers receive) must run concurrently with the
# trainer-side broadcasts — both rendezvous inside the same NCCL calls.
exe = ThreadPoolExecutor(max_workers=1)
try:
future = exe.submit(self.client.update_weights, asdict(update_info))
# Cheap best-effort: if update_weights already failed (e.g. a bad
# request rejected before any NCCL call), surface it now instead of
# hanging in broadcast waiting for a peer that will never arrive.
if future.done():
future.result()
stream = torch.cuda.current_stream()
for patch in patches:
self.model_update_group.broadcast(patch.indices, src=0, stream=stream)
self.model_update_group.broadcast(patch.values, src=0, stream=stream)
future.result() # surface inference-side errors
finally:
# Never wait for the RPC thread here: if a broadcast raised, the
# worker is still blocked in the matching NCCL call and will never
# return, so joining would turn the error into a permanent hang.
# See NCCLTrainerWeightTransferEngine.send_weights.
exe.shutdown(wait=False)
self.client.finish_weight_update()
self._post_send_sync()
@staticmethod
def _validate_patch(patch: SparseWeightPatch) -> None:
"""Reject a malformed patch before any NCCL call.
The worker checks the same invariants in `_apply_patch`, but by then the
broadcasts are already under way: a mismatch surfaces as a size mismatch
mid-transfer, which wedges both sides instead of raising. Checking here
keeps the failure on the trainer, before `start_weight_update`.
"""
if patch.indices.dtype != torch.int32:
raise ValueError(
f"Sparse weight updates require int32 indices: {patch.name}"
)
if patch.indices.ndim != 1 or patch.values.ndim != 1:
raise ValueError(
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
)
if patch.indices.numel() != patch.values.numel():
raise ValueError(
f"`indices` and `values` must have matching lengths for {patch.name}"
)
stream = args.stream or torch.cuda.current_stream()
for patch in iterator:
args.group.broadcast(patch.indices, src=args.src, stream=stream)
args.group.broadcast(patch.values, src=args.src, stream=stream)
def _post_send_sync(self) -> None:
"""Wait for the broadcasts to land before returning, so a caller may
rebuild or free the patch tensors as soon as `send_weights` returns
rather than relying on same-stream ordering. See
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
cross-rank barrier (and sparse is single-rank on the trainer anyway)."""
if torch.cuda.is_available():
torch.cuda.current_stream().synchronize()
# Trainer-side process-group setup (shared with the dense engine).
trainer_init = staticmethod(trainer_init)
def shutdown(self) -> None:
self.model_update_group = None