[rl] Stateful Trainer Send: NCCL + Sparse NCCL [3/N] (#50902)
Signed-off-by: haoaaron <ahao@anyscale.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
43d691ec6b
commit
21ea5b4fa1
@@ -77,6 +77,7 @@ class WeightSource(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def metadata(self) -> list[ParamMeta]:
|
||||
"""Declare what iteration will yield, without transferring anything."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
@@ -402,8 +403,7 @@ class TrainerWeightTransferEngine(ABC, Generic[TTrainerInitInfo]):
|
||||
Symmetric to `WeightTransferEngine` but lives in the training process.
|
||||
Constructed via the `trainer_init` factory classmethod; carries any
|
||||
backend-specific state (NCCL communicators, IPC device info, transfer
|
||||
plans) on `self`. The `WeightSource` is required at `trainer_init`,
|
||||
then replayed each round by the no-argument `send_weights()`.
|
||||
plans) on `self`.
|
||||
|
||||
Unlike the worker engine, the trainer side does not take a
|
||||
`WeightTransferConfig`: the backend is selected from the init info's
|
||||
@@ -431,7 +431,7 @@ class TrainerWeightTransferEngine(ABC, Generic[TTrainerInitInfo]):
|
||||
self,
|
||||
*,
|
||||
client: "VLLMWeightSyncClient",
|
||||
source: "WeightSource",
|
||||
source: "WeightSource | None" = None,
|
||||
is_sender: bool = True,
|
||||
) -> None:
|
||||
self.is_sender = is_sender
|
||||
@@ -447,24 +447,24 @@ class TrainerWeightTransferEngine(ABC, Generic[TTrainerInitInfo]):
|
||||
init_info: TTrainerInitInfo,
|
||||
*,
|
||||
client: "VLLMWeightSyncClient",
|
||||
source: "WeightSource",
|
||||
source: "WeightSource | None" = None,
|
||||
) -> Self:
|
||||
"""Rendezvous with the inference side and return a ready instance.
|
||||
|
||||
Called on every trainer rank. The sender drives the full handshake via
|
||||
`client` (build the worker-side init info, call
|
||||
`client.init_weight_transfer_engine`, open the trainer-side endpoint);
|
||||
non-sender ranks skip the rendezvous and the RPC. `source` is stored on
|
||||
`self.source`; after return, `send_weights()` is callable.
|
||||
non-sender ranks skip the rendezvous and the RPC.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def send_weights(self) -> None:
|
||||
"""Push `self.source`'s weights to inference workers and drive the full
|
||||
update round trip: `start_weight_update`, `update_weights` (run
|
||||
concurrently with the trainer-side broadcast when the backend requires
|
||||
it), then `finish_weight_update`. Called on every trainer rank."""
|
||||
"""Push weights to inference workers and drive the full update round
|
||||
trip: `start_weight_update`, `update_weights` (run concurrently with the
|
||||
trainer-side broadcast when the backend requires it), then
|
||||
`finish_weight_update`. Called on every trainer rank.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def shutdown(self) -> None:
|
||||
|
||||
@@ -167,7 +167,7 @@ class WeightTransferTrainerFactory:
|
||||
init_info: "TrainerInitInfo",
|
||||
*,
|
||||
client: "VLLMWeightSyncClient",
|
||||
source: "WeightSource",
|
||||
source: "WeightSource | None" = None,
|
||||
) -> TrainerWeightTransferEngine:
|
||||
"""Build and rendezvous a ready-to-send trainer engine.
|
||||
|
||||
@@ -184,7 +184,9 @@ class WeightTransferTrainerFactory:
|
||||
selects the engine; it also carries the wire params (e.g.
|
||||
`packed`).
|
||||
client: Inference-side control-plane client.
|
||||
source: `WeightSource` of `(name, tensor)` pairs to send each round.
|
||||
source: `WeightSource` of `(name, tensor)` pairs to send each round,
|
||||
for full-resync backends (NCCL, IPC). Sparse backend
|
||||
omits it and passes its per-round payload to `send_weights`.
|
||||
|
||||
Raises:
|
||||
ValueError: If `init_info.backend` is not registered.
|
||||
@@ -233,10 +235,21 @@ WeightTransferEngineFactory.register_engine(
|
||||
)
|
||||
|
||||
|
||||
# Trainer-side engines. Backends register here as they migrate to the stateful
|
||||
# trainer engine; NCCL / sparse NCCL keep their static trainer path until then.
|
||||
# Trainer-side engines, parallel to the worker registry above.
|
||||
WeightTransferTrainerFactory.register_engine(
|
||||
"nccl",
|
||||
"vllm.distributed.weight_transfer.nccl_engine",
|
||||
"NCCLTrainerWeightTransferEngine",
|
||||
)
|
||||
|
||||
WeightTransferTrainerFactory.register_engine(
|
||||
"ipc",
|
||||
"vllm.distributed.weight_transfer.ipc_engine",
|
||||
"IPCTrainerWeightTransferEngine",
|
||||
)
|
||||
|
||||
WeightTransferTrainerFactory.register_engine(
|
||||
"sparse_nccl",
|
||||
"vllm.distributed.weight_transfer.sparse_nccl_engine",
|
||||
"SparseNCCLTrainerWeightTransferEngine",
|
||||
)
|
||||
|
||||
@@ -291,8 +291,10 @@ class IPCTrainerWeightTransferEngine(TrainerWeightTransferEngine[IPCTrainerInitI
|
||||
init_info: IPCTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
if source is None:
|
||||
raise ValueError("IPC trainer weight transfer requires a WeightSource.")
|
||||
engine = cls(
|
||||
client=client,
|
||||
source=source,
|
||||
@@ -307,6 +309,7 @@ class IPCTrainerWeightTransferEngine(TrainerWeightTransferEngine[IPCTrainerInitI
|
||||
return engine
|
||||
|
||||
def send_weights(self) -> None:
|
||||
assert self.source is not None
|
||||
source = self.source
|
||||
if self.is_sender:
|
||||
self.client.start_weight_update()
|
||||
|
||||
@@ -9,7 +9,7 @@ sparse engine does not have to subclass the dense one.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
import torch
|
||||
|
||||
@@ -18,16 +18,36 @@ if TYPE_CHECKING:
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
DEFAULT_PACKED_NUM_BUFFERS,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Initialization info for NCCL-based weight transfer backends."""
|
||||
"""Worker-side initialization info for NCCL-based weight transfer backends."""
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
rank_offset: int
|
||||
world_size: int
|
||||
packed: bool = False
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
|
||||
|
||||
|
||||
class NCCLRendezvous(Protocol):
|
||||
"""The three fields `trainer_init` needs to open the trainer endpoint.
|
||||
|
||||
Structural so each backend can keep its own trainer init info next to its
|
||||
engine (dense `NCCLTrainerInitInfo`, `SparseNCCLTrainerInitInfo`) without
|
||||
this shared module importing either.
|
||||
"""
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
|
||||
|
||||
def stateless_init_process_group(
|
||||
@@ -83,7 +103,7 @@ def worker_init_process_group(
|
||||
|
||||
|
||||
def trainer_init(
|
||||
init_info: NCCLWeightTransferInitInfo | dict,
|
||||
init_info: NCCLRendezvous | dict,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""
|
||||
Initialize NCCL process group for trainer-side weight transfer.
|
||||
@@ -92,7 +112,8 @@ def trainer_init(
|
||||
CUDA device (torch.accelerator.current_device_index()).
|
||||
|
||||
Args:
|
||||
init_info: Either an NCCLWeightTransferInitInfo object or a dict with keys:
|
||||
init_info: Any object carrying the `NCCLRendezvous` fields (a trainer or
|
||||
worker NCCL init info), or a dict with keys:
|
||||
- master_address: str
|
||||
- master_port: int
|
||||
- world_size: int
|
||||
|
||||
@@ -2,11 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NCCL-based (dense) weight transfer engine."""
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from collections.abc import Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import torch
|
||||
from typing_extensions import Self
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
@@ -14,72 +16,76 @@ if TYPE_CHECKING:
|
||||
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
ParamMeta,
|
||||
TrainerInitInfo,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
WeightTransferEngine,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
trainer_init,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
trainer_init as open_trainer_endpoint,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
DEFAULT_PACKED_NUM_BUFFERS,
|
||||
packed_nccl_broadcast_consumer,
|
||||
packed_nccl_broadcast_producer,
|
||||
)
|
||||
|
||||
# Re-exported for backward compatibility; canonical home is nccl_common.
|
||||
# NCCLWeightTransferInitInfo is re-exported here for convenience; its canonical
|
||||
# home is nccl_common, shared with the sparse backend.
|
||||
__all__ = [
|
||||
"NCCLWeightTransferInitInfo",
|
||||
"NCCLTrainerSendWeightsArgs",
|
||||
"NCCLTrainerInitInfo",
|
||||
"NCCLWeightTransferUpdateInfo",
|
||||
"NCCLWeightTransferEngine",
|
||||
"NCCLTrainerWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLTrainerSendWeightsArgs:
|
||||
"""Arguments for NCCL trainer_send_weights method."""
|
||||
class NCCLTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for the dense NCCL weight transfer backend.
|
||||
|
||||
group: Any
|
||||
"""Process group (PyNcclCommunicator) for NCCL communication."""
|
||||
src: int = 0
|
||||
"""Source rank (default 0, trainer is typically rank 0)."""
|
||||
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor] | None = None
|
||||
"""Optional function to apply to each (name, tensor) pair before broadcasting.
|
||||
If None, extracts just the tensor."""
|
||||
packed: bool = False
|
||||
"""Whether to use packed tensor broadcasting for efficiency.
|
||||
When True, multiple tensors are batched together before broadcasting
|
||||
to reduce NCCL communication overhead."""
|
||||
stream: torch.cuda.Stream | None = None
|
||||
"""CUDA stream to use for broadcasting if packed is False.
|
||||
If packed is True, new streams will be created for each buffer."""
|
||||
The sender opens its endpoint as NCCL rank 0, so it needs no `rank_offset`.
|
||||
`world_size` is the full trainer+worker NCCL group size. `rank` (from
|
||||
`TrainerInitInfo`) identifies this trainer process; rank 0 is the sender.
|
||||
|
||||
`packed` / buffer sizes are the transfer's wire params. The trainer
|
||||
propagates them to the worker at `trainer_init` so the two sides cannot
|
||||
disagree. Note this defaults to packed, unlike the worker-side
|
||||
`NCCLWeightTransferInitInfo`, whose default only applies when no trainer
|
||||
ships a value. `backend` is the factory dispatch key."""
|
||||
|
||||
backend: ClassVar[str] = "nccl"
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
packed: bool = True
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
"""Size in bytes for each packed tensor buffer.
|
||||
Must match the value used in NCCLWeightTransferUpdateInfo."""
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
|
||||
"""Number of buffers for double/triple buffering during packed transfer.
|
||||
Must match the value used in NCCLWeightTransferUpdateInfo."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Update info for NCCL weight transfer backend."""
|
||||
"""Per-round update info for the dense NCCL weight transfer backend.
|
||||
|
||||
Whether the transfer is packed (and the buffer geometry) is a must-agree
|
||||
wire param carried on the init info (`NCCLTrainerInitInfo` /
|
||||
`NCCLWeightTransferInitInfo`), not here; this carries only the per-round
|
||||
parameter metadata.
|
||||
"""
|
||||
|
||||
names: list[str]
|
||||
dtype_names: list[str]
|
||||
shapes: list[list[int]]
|
||||
packed: bool = False
|
||||
"""Whether to use packed tensor broadcasting for efficiency.
|
||||
When True, multiple tensors are batched together before broadcasting
|
||||
to reduce NCCL communication overhead."""
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
"""Size in bytes for each packed tensor buffer.
|
||||
Both producer and consumer must use the same value."""
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
|
||||
"""Number of buffers for double/triple buffering during packed transfer.
|
||||
Both producer and consumer must use the same value."""
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate that all lists have the same length."""
|
||||
@@ -121,15 +127,25 @@ class NCCLWeightTransferEngine(
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
# Set from the trainer-supplied init info at the handshake; defaults are
|
||||
# only for the (unreachable) receive-before-init case.
|
||||
self.packed = False
|
||||
self.packed_buffer_size_bytes = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
self.packed_num_buffers = DEFAULT_PACKED_NUM_BUFFERS
|
||||
|
||||
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
|
||||
"""
|
||||
Initialize NCCL process group with the trainer.
|
||||
Initialize NCCL process group with the trainer and record the
|
||||
trainer-supplied wire params so the worker decodes exactly as the
|
||||
trainer encodes.
|
||||
|
||||
Args:
|
||||
init_info: NCCL initialization info containing master address, port,
|
||||
rank offset, and world size
|
||||
rank offset, world size, and the packed wire params
|
||||
"""
|
||||
self.packed = init_info.packed
|
||||
self.packed_buffer_size_bytes = init_info.packed_buffer_size_bytes
|
||||
self.packed_num_buffers = init_info.packed_num_buffers
|
||||
self.model_update_group = worker_init_process_group(
|
||||
init_info, self.parallel_config
|
||||
)
|
||||
@@ -154,13 +170,14 @@ class NCCLWeightTransferEngine(
|
||||
"""
|
||||
Receive weights from trainer via NCCL broadcast.
|
||||
|
||||
If update_info.packed is True, uses packed tensor broadcasting for
|
||||
efficient transfer of multiple weights in batches. Otherwise, uses simple
|
||||
one-by-one broadcasting.
|
||||
Whether to use packed broadcasting (and the buffer geometry) is read
|
||||
from `self.packed` / `self.packed_*`, set at the init handshake from the
|
||||
trainer's init info, so it is guaranteed to match how the trainer
|
||||
encoded.
|
||||
|
||||
Args:
|
||||
update_info: NCCL update info containing parameter names, dtypes, shapes,
|
||||
and packed flag
|
||||
update_info: NCCL update info containing parameter names, dtypes,
|
||||
and shapes
|
||||
"""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
@@ -173,7 +190,7 @@ class NCCLWeightTransferEngine(
|
||||
)
|
||||
|
||||
with disable_mtp_completeness_check():
|
||||
if update_info.packed:
|
||||
if self.packed:
|
||||
# Build iterator of (name, (shape, dtype)) from update_info
|
||||
def state_dict_info_iterator():
|
||||
for name, dtype_name, shape in zip(
|
||||
@@ -187,8 +204,8 @@ class NCCLWeightTransferEngine(
|
||||
group=self.model_update_group,
|
||||
src=0,
|
||||
post_unpack_func=self.model.load_weights,
|
||||
buffer_size_bytes=update_info.packed_buffer_size_bytes,
|
||||
num_buffers=update_info.packed_num_buffers,
|
||||
buffer_size_bytes=self.packed_buffer_size_bytes,
|
||||
num_buffers=self.packed_num_buffers,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
@@ -210,63 +227,241 @@ class NCCLWeightTransferEngine(
|
||||
self.model_update_group = None
|
||||
|
||||
@staticmethod
|
||||
def trainer_send_weights(
|
||||
iterator: Iterator[tuple[str, torch.Tensor]],
|
||||
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
|
||||
) -> None:
|
||||
"""Broadcast dense weights from trainer to vLLM workers.
|
||||
def trainer_send_weights(*args: Any, **kwargs: Any) -> None:
|
||||
"""Removed. Use the stateful `NCCLTrainerWeightTransferEngine` instead.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of model parameters. Returns (name, tensor) tuples
|
||||
trainer_args: Dictionary or NCCLTrainerSendWeightsArgs instance containing
|
||||
NCCL-specific arguments. If a dict, should contain keys from
|
||||
NCCLTrainerSendWeightsArgs.
|
||||
|
||||
Example:
|
||||
>>> from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
... NCCLWeightTransferEngine,
|
||||
... NCCLTrainerSendWeightsArgs,
|
||||
... )
|
||||
>>> param_iter = ((n, p) for n, p in model.named_parameters())
|
||||
>>> args = NCCLTrainerSendWeightsArgs(group=group, packed=True)
|
||||
>>> NCCLWeightTransferEngine.trainer_send_weights(param_iter, args)
|
||||
Transitional stub kept only to satisfy the (still abstract)
|
||||
`WeightTransferEngine.trainer_send_weights`; that member is dropped from
|
||||
the worker ABC once every backend has migrated to the trainer engine.
|
||||
"""
|
||||
# Parse trainer args - accept either dict or dataclass instance
|
||||
if isinstance(trainer_args, dict):
|
||||
args = NCCLTrainerSendWeightsArgs(**trainer_args)
|
||||
else:
|
||||
args = trainer_args
|
||||
raise NotImplementedError(
|
||||
"The static NCCL trainer path has been replaced by "
|
||||
"NCCLTrainerWeightTransferEngine. Build it via "
|
||||
"WeightTransferTrainerFactory.trainer_init(NCCLTrainerInitInfo(...), "
|
||||
"client=..., source=...) and drive it with send_weights()."
|
||||
)
|
||||
|
||||
if args.post_iter_func is None:
|
||||
# Default: extract just the tensor from (name, tensor) tuple
|
||||
post_iter_func = lambda x: x[1]
|
||||
else:
|
||||
post_iter_func = args.post_iter_func
|
||||
|
||||
if args.packed:
|
||||
# Use packed tensor broadcasting for efficiency
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
packed_nccl_broadcast_producer,
|
||||
class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerInitInfo]):
|
||||
"""Trainer-side NCCL weight transfer engine.
|
||||
|
||||
On the sender (rank 0) holds the NCCL communicator and drives the full
|
||||
update round trip: it runs the inference-side `update_weights` concurrently
|
||||
with the trainer-side broadcast (both rendezvous inside the same NCCL
|
||||
calls), then finishes the update. Non-sender trainer ranks hold no
|
||||
communicator; they only iterate the source to stay in the trainer-side
|
||||
collective (e.g. FSDP `full_tensor()`) and skip the client RPCs and the
|
||||
broadcast (all guarded on `is_sender`).
|
||||
|
||||
`packed` / buffer sizes come from `NCCLTrainerInitInfo`; the sender
|
||||
propagates them to the worker at `trainer_init` (on the worker-side init
|
||||
info), so per-round payloads carry only parameter metadata.
|
||||
"""
|
||||
|
||||
init_info_cls = NCCLTrainerInitInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource,
|
||||
is_sender: bool = True,
|
||||
packed: bool = True,
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
|
||||
) -> None:
|
||||
super().__init__(client=client, source=source, is_sender=is_sender)
|
||||
self.packed = packed
|
||||
self.packed_buffer_size_bytes = packed_buffer_size_bytes
|
||||
self.packed_num_buffers = packed_num_buffers
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
init_info: NCCLTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
if source is None:
|
||||
raise ValueError("NCCL trainer weight transfer requires a WeightSource.")
|
||||
engine = cls(
|
||||
client=client,
|
||||
source=source,
|
||||
is_sender=init_info.is_sender,
|
||||
packed=init_info.packed,
|
||||
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
|
||||
packed_num_buffers=init_info.packed_num_buffers,
|
||||
)
|
||||
if not engine.is_sender:
|
||||
# Non-sender trainer ranks aren't part of the transfer NCCL group and
|
||||
# don't drive the inference side; they only participate in the
|
||||
# trainer-side gather during send_weights.
|
||||
return engine
|
||||
|
||||
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
|
||||
worker_init_info = NCCLWeightTransferInitInfo(
|
||||
master_address=init_info.master_address,
|
||||
master_port=init_info.master_port,
|
||||
rank_offset=1,
|
||||
world_size=init_info.world_size,
|
||||
packed=init_info.packed,
|
||||
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
|
||||
packed_num_buffers=init_info.packed_num_buffers,
|
||||
)
|
||||
|
||||
# The inference workers block inside init_weight_transfer_engine waiting
|
||||
# for the NCCL rendezvous, so we kick that off on a side thread while we
|
||||
# open the trainer endpoint (rank 0); both sides must rendezvous together.
|
||||
with ThreadPoolExecutor(max_workers=1) as exe:
|
||||
future = exe.submit(
|
||||
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
|
||||
)
|
||||
engine.model_update_group = open_trainer_endpoint(init_info)
|
||||
future.result() # surface any inference-side init error
|
||||
|
||||
return engine
|
||||
|
||||
def send_weights(self) -> None:
|
||||
assert self.source is not None # guaranteed by trainer_init / __init__
|
||||
source = self.source
|
||||
|
||||
# Metadata is declared without gathering. For Megatron it is itself a
|
||||
# collective, so every rank runs it; only the sender ships it.
|
||||
meta = source.metadata()
|
||||
|
||||
if not self.is_sender:
|
||||
# Non-sender ranks only join the trainer-side gather collective.
|
||||
self._broadcast(source, meta)
|
||||
self._post_send_sync()
|
||||
return
|
||||
|
||||
update_info = NCCLWeightTransferUpdateInfo(
|
||||
names=[m.name for m in meta],
|
||||
dtype_names=[str(m.dtype).split(".")[-1] for m in meta],
|
||||
shapes=[list(m.shape) for m in meta],
|
||||
)
|
||||
|
||||
self.client.start_weight_update()
|
||||
# update_weights (workers receive) must run concurrently with the
|
||||
# trainer-side broadcast — both rendezvous inside the same NCCL calls.
|
||||
exe = ThreadPoolExecutor(max_workers=1)
|
||||
try:
|
||||
future = exe.submit(self.client.update_weights, asdict(update_info))
|
||||
# Cheap best-effort: if update_weights already failed (e.g. a bad
|
||||
# request rejected before any NCCL call), surface it now instead of
|
||||
# hanging in broadcast waiting for a peer that will never arrive.
|
||||
if future.done():
|
||||
future.result()
|
||||
self._broadcast(source, meta)
|
||||
future.result() # surface inference-side errors
|
||||
finally:
|
||||
# Never wait for the RPC thread here. If the broadcast raised, the
|
||||
# worker is still blocked in the matching NCCL call and will never
|
||||
# return, so joining would turn the error into a permanent hang.
|
||||
# Let the exception out instead; the transfer group is unusable
|
||||
# either way and the caller has to tear it down. (The orphaned
|
||||
# thread is still joined at interpreter exit, so a caller that wants
|
||||
# to exit cleanly after such a failure must not wait on it.)
|
||||
exe.shutdown(wait=False)
|
||||
self.client.finish_weight_update()
|
||||
self._post_send_sync()
|
||||
|
||||
def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None:
|
||||
"""Iterate the source (materializing each tensor — a collective on all
|
||||
ranks) and, on the sender, broadcast from rank 0, packed or one-by-one.
|
||||
Non-sender ranks only replay the iteration to stay in the collective."""
|
||||
if not self.is_sender:
|
||||
for _ in source:
|
||||
pass
|
||||
return
|
||||
|
||||
assert self.model_update_group is not None, (
|
||||
"trainer_init() must be called before _broadcast()."
|
||||
)
|
||||
pairs = self._checked_iter(source, meta)
|
||||
if self.packed:
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=iterator,
|
||||
group=args.group,
|
||||
src=args.src,
|
||||
post_iter_func=post_iter_func,
|
||||
buffer_size_bytes=args.packed_buffer_size_bytes,
|
||||
num_buffers=args.packed_num_buffers,
|
||||
iterator=pairs,
|
||||
group=self.model_update_group,
|
||||
src=0,
|
||||
post_iter_func=lambda item: item[1],
|
||||
buffer_size_bytes=self.packed_buffer_size_bytes,
|
||||
num_buffers=self.packed_num_buffers,
|
||||
)
|
||||
else:
|
||||
# Use simple one-by-one broadcasting
|
||||
for item in iterator:
|
||||
tensor = post_iter_func(item)
|
||||
args.group.broadcast(
|
||||
tensor,
|
||||
src=args.src,
|
||||
stream=args.stream or torch.cuda.current_stream(),
|
||||
)
|
||||
stream = torch.cuda.current_stream()
|
||||
for _name, tensor in pairs:
|
||||
# NCCL sends `numel` elements straight from `data_ptr()`, so a
|
||||
# non-contiguous view would ship whatever follows its base
|
||||
# pointer. Keep the copy referenced until the broadcast is
|
||||
# enqueued. (The packed path linearizes in `pack_tensors`.)
|
||||
send = tensor if tensor.is_contiguous() else tensor.contiguous()
|
||||
self.model_update_group.broadcast(send, src=0, stream=stream)
|
||||
|
||||
# Trainer-side process-group setup. Delegates to the shared helper so the
|
||||
# sparse engine can reuse the exact same rendezvous without subclassing.
|
||||
trainer_init = staticmethod(trainer_init)
|
||||
@staticmethod
|
||||
def _checked_iter(
|
||||
source: WeightSource, meta: list[ParamMeta]
|
||||
) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""Yield the source's pairs, checking each against what the worker was
|
||||
told to expect.
|
||||
|
||||
The worker sizes its receive buffers — and in packed mode cuts its chunk
|
||||
boundaries — from the update info, which is built from `metadata()`. If
|
||||
iteration disagrees with it, the two sides split the stream differently
|
||||
and the transfer hangs in NCCL or loads garbage. Checking here costs one
|
||||
comparison per parameter and turns that into an error naming the first
|
||||
divergent parameter. Sender-only: under pipeline parallelism a
|
||||
non-sender's yielded tensor is not meaningful.
|
||||
"""
|
||||
sent = 0
|
||||
for name, tensor in source:
|
||||
if sent >= len(meta):
|
||||
raise ValueError(
|
||||
f"WeightSource yielded more parameters than metadata() "
|
||||
f"declared ({len(meta)}); first extra is {name!r}."
|
||||
)
|
||||
expected = meta[sent]
|
||||
if (
|
||||
name != expected.name
|
||||
or tensor.dtype != expected.dtype
|
||||
or tuple(tensor.shape) != expected.shape
|
||||
):
|
||||
raise ValueError(
|
||||
"WeightSource metadata() disagrees with iteration at index "
|
||||
f"{sent}: declared {expected.name!r} "
|
||||
f"{expected.dtype} {tuple(expected.shape)}, got {name!r} "
|
||||
f"{tensor.dtype} {tuple(tensor.shape)}. Both channels must "
|
||||
"enumerate the same parameters in the same order."
|
||||
)
|
||||
sent += 1
|
||||
yield name, tensor
|
||||
if sent != len(meta):
|
||||
raise ValueError(
|
||||
f"WeightSource yielded {sent} parameters but metadata() "
|
||||
f"declared {len(meta)}; the worker is waiting for the rest."
|
||||
)
|
||||
|
||||
def _post_send_sync(self) -> None:
|
||||
"""Wait for this rank's transfer work to land before returning.
|
||||
|
||||
Broadcasts are only *enqueued* by `send_weights`: the unpacked path on
|
||||
the current stream, the packed path on the producer's own streams (which
|
||||
it drains itself). Waiting here lets a caller mutate parameters, or
|
||||
start the next step on another stream, as soon as `send_weights`
|
||||
returns, instead of silently depending on same-stream ordering. Every
|
||||
rank waits: a non-sender's `full_tensor()` gathers feed the sender's
|
||||
broadcast, so they must have landed before it may touch its shards.
|
||||
|
||||
Unlike IPC there is no cross-rank barrier here. Nothing in this backend
|
||||
outlives the collective it travelled in (IPC's barrier keeps shared
|
||||
buffers alive until every consumer has opened them), so a barrier would
|
||||
only add a dependency on the default process group that this backend
|
||||
otherwise does not have.
|
||||
"""
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.model_update_group = None
|
||||
|
||||
@@ -177,6 +177,10 @@ def packed_nccl_broadcast_producer(
|
||||
# Move to the next buffer
|
||||
buffer_idx = (buffer_idx + 1) % num_buffers
|
||||
|
||||
# Drain every slot before returning.
|
||||
for stream in streams:
|
||||
stream.synchronize()
|
||||
|
||||
|
||||
def packed_nccl_broadcast_consumer(
|
||||
iterator: Iterator[tuple[str, tuple[list[int], torch.dtype]]],
|
||||
@@ -279,6 +283,10 @@ def packed_nccl_broadcast_consumer(
|
||||
)
|
||||
break
|
||||
|
||||
# Drain every slot before returning.
|
||||
for stream in streams:
|
||||
stream.synchronize()
|
||||
|
||||
|
||||
# ── IPC packed transfer ────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -14,11 +14,13 @@ MVP limitations:
|
||||
* not composable with checkpoint-format or packed updates
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from collections.abc import Iterable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import torch
|
||||
from typing_extensions import Self
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
@@ -26,20 +28,27 @@ if TYPE_CHECKING:
|
||||
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
TrainerInitInfo,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
WeightTransferEngine,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
trainer_init,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerSendWeightsArgs
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
trainer_init as open_trainer_endpoint,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SparseWeightPatch",
|
||||
"SparseNCCLTrainerInitInfo",
|
||||
"SparseNCCLWeightTransferUpdateInfo",
|
||||
"SparseNCCLWeightTransferEngine",
|
||||
"SparseNCCLTrainerWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
@@ -50,6 +59,25 @@ class SparseWeightPatch:
|
||||
name: str
|
||||
indices: torch.Tensor
|
||||
values: torch.Tensor
|
||||
full_shape: tuple[int, ...] | None = None
|
||||
"""Full shape of the patched parameter. Required when the patch is sent
|
||||
via `SparseNCCLTrainerWeightTransferEngine` (it ships in the per-round
|
||||
update info); the worker-side apply path does not read it."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseNCCLTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for the sparse NCCL weight transfer backend.
|
||||
|
||||
Same rendezvous shape as the dense NCCL backend (the sender opens its
|
||||
endpoint as NCCL rank 0), but with no packed wire params: sparse transfers
|
||||
are never packed. `backend` is the factory dispatch key."""
|
||||
|
||||
backend: ClassVar[str] = "sparse_nccl"
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -200,25 +228,182 @@ class SparseNCCLWeightTransferEngine(
|
||||
self.model_update_group = None
|
||||
|
||||
@staticmethod
|
||||
def trainer_send_weights(
|
||||
iterator: Iterator[SparseWeightPatch],
|
||||
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
|
||||
) -> None:
|
||||
"""Broadcast sparse flat-index patches from trainer to vLLM workers."""
|
||||
if isinstance(trainer_args, dict):
|
||||
args = NCCLTrainerSendWeightsArgs(**trainer_args)
|
||||
else:
|
||||
args = trainer_args
|
||||
def trainer_send_weights(*args: Any, **kwargs: Any) -> None:
|
||||
"""Removed. Use the stateful `SparseNCCLTrainerWeightTransferEngine`.
|
||||
|
||||
if args.packed:
|
||||
Transitional stub kept only to satisfy the (still abstract)
|
||||
`WeightTransferEngine.trainer_send_weights`; that member is dropped from
|
||||
the worker ABC once every backend has migrated to the trainer engine.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"The static sparse NCCL trainer path has been replaced by "
|
||||
"SparseNCCLTrainerWeightTransferEngine. Build it via "
|
||||
"WeightTransferTrainerFactory.trainer_init("
|
||||
"SparseNCCLTrainerInitInfo(...), client=..., source=...) and drive "
|
||||
"it with send_weights()."
|
||||
)
|
||||
|
||||
|
||||
class SparseNCCLTrainerWeightTransferEngine(
|
||||
TrainerWeightTransferEngine[SparseNCCLTrainerInitInfo]
|
||||
):
|
||||
"""Trainer-side sparse NCCL weight transfer engine.
|
||||
|
||||
Broadcasts flat-index (indices, values) patches from NCCL rank 0 while the
|
||||
inference-side `update_weights` runs concurrently on a side thread (the
|
||||
worker's recvs rendezvous inside the same NCCL broadcasts), then finishes
|
||||
the update. Unlike the full-resync backends, sparse patches differ every
|
||||
round (a fresh set of deltas from each optimizer step), so they are not a
|
||||
stable `WeightSource`: the engine takes no `source`, and each round's
|
||||
patches are passed straight to `send_weights(patches)`. A round with no
|
||||
patches is a no-op.
|
||||
|
||||
The sparse backend assumes a single-rank trainer (matching its TP=1 / PP=1
|
||||
MVP scope), so non-sender ranks skip `send_weights` entirely.
|
||||
"""
|
||||
|
||||
init_info_cls = SparseNCCLTrainerInitInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
is_sender: bool = True,
|
||||
) -> None:
|
||||
# Sparse is a delta backend: it takes per-round patches via
|
||||
# send_weights, so a `source` would silently never be sent. The
|
||||
# parameter exists only to match the base/factory signature.
|
||||
if source is not None:
|
||||
raise ValueError(
|
||||
"Sparse NCCL updates cannot be combined with `packed=True`"
|
||||
"Sparse NCCL weight transfer takes no WeightSource; pass each "
|
||||
"round's patches to send_weights(patches) instead."
|
||||
)
|
||||
super().__init__(client=client, source=source, is_sender=is_sender)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
init_info: SparseNCCLTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
engine = cls(client=client, source=source, is_sender=init_info.is_sender)
|
||||
if not engine.is_sender:
|
||||
return engine
|
||||
|
||||
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
|
||||
# Sparse transfers are never packed, so the worker keeps the unpacked
|
||||
# defaults on its init info.
|
||||
worker_init_info = NCCLWeightTransferInitInfo(
|
||||
master_address=init_info.master_address,
|
||||
master_port=init_info.master_port,
|
||||
rank_offset=1,
|
||||
world_size=init_info.world_size,
|
||||
)
|
||||
|
||||
# The inference workers block inside init_weight_transfer_engine waiting
|
||||
# for the NCCL rendezvous, so we kick that off on a side thread while we
|
||||
# open the trainer endpoint (rank 0); both sides must rendezvous together.
|
||||
with ThreadPoolExecutor(max_workers=1) as exe:
|
||||
future = exe.submit(
|
||||
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
|
||||
)
|
||||
# Open the trainer endpoint as NCCL rank 0 on the current device
|
||||
# (the init info satisfies the helper's rendezvous protocol).
|
||||
engine.model_update_group = open_trainer_endpoint(init_info)
|
||||
future.result() # surface any inference-side init error
|
||||
|
||||
return engine
|
||||
|
||||
def send_weights(self, patches: Iterable[SparseWeightPatch] | None = None) -> None:
|
||||
"""Broadcast this round's sparse patches. `patches` is the per-round
|
||||
payload (sparse deltas differ every round), so it is passed here rather
|
||||
than fixed at init. Every patch must set `full_shape`."""
|
||||
if not self.is_sender:
|
||||
return
|
||||
|
||||
patches = list(patches) if patches is not None else []
|
||||
if not patches:
|
||||
return
|
||||
|
||||
shapes = []
|
||||
for patch in patches:
|
||||
if patch.full_shape is None:
|
||||
raise ValueError(
|
||||
"SparseWeightPatch.full_shape must be set to send via the "
|
||||
f"trainer engine: {patch.name}"
|
||||
)
|
||||
self._validate_patch(patch)
|
||||
shapes.append(list(patch.full_shape))
|
||||
|
||||
update_info = SparseNCCLWeightTransferUpdateInfo(
|
||||
names=[patch.name for patch in patches],
|
||||
dtype_names=[str(patch.values.dtype).split(".")[-1] for patch in patches],
|
||||
shapes=shapes,
|
||||
num_updates_list=[patch.indices.numel() for patch in patches],
|
||||
)
|
||||
|
||||
assert self.model_update_group is not None, (
|
||||
"trainer_init() must be called before send_weights()."
|
||||
)
|
||||
self.client.start_weight_update()
|
||||
# update_weights (workers receive) must run concurrently with the
|
||||
# trainer-side broadcasts — both rendezvous inside the same NCCL calls.
|
||||
exe = ThreadPoolExecutor(max_workers=1)
|
||||
try:
|
||||
future = exe.submit(self.client.update_weights, asdict(update_info))
|
||||
# Cheap best-effort: if update_weights already failed (e.g. a bad
|
||||
# request rejected before any NCCL call), surface it now instead of
|
||||
# hanging in broadcast waiting for a peer that will never arrive.
|
||||
if future.done():
|
||||
future.result()
|
||||
stream = torch.cuda.current_stream()
|
||||
for patch in patches:
|
||||
self.model_update_group.broadcast(patch.indices, src=0, stream=stream)
|
||||
self.model_update_group.broadcast(patch.values, src=0, stream=stream)
|
||||
future.result() # surface inference-side errors
|
||||
finally:
|
||||
# Never wait for the RPC thread here: if a broadcast raised, the
|
||||
# worker is still blocked in the matching NCCL call and will never
|
||||
# return, so joining would turn the error into a permanent hang.
|
||||
# See NCCLTrainerWeightTransferEngine.send_weights.
|
||||
exe.shutdown(wait=False)
|
||||
self.client.finish_weight_update()
|
||||
self._post_send_sync()
|
||||
|
||||
@staticmethod
|
||||
def _validate_patch(patch: SparseWeightPatch) -> None:
|
||||
"""Reject a malformed patch before any NCCL call.
|
||||
|
||||
The worker checks the same invariants in `_apply_patch`, but by then the
|
||||
broadcasts are already under way: a mismatch surfaces as a size mismatch
|
||||
mid-transfer, which wedges both sides instead of raising. Checking here
|
||||
keeps the failure on the trainer, before `start_weight_update`.
|
||||
"""
|
||||
if patch.indices.dtype != torch.int32:
|
||||
raise ValueError(
|
||||
f"Sparse weight updates require int32 indices: {patch.name}"
|
||||
)
|
||||
if patch.indices.ndim != 1 or patch.values.ndim != 1:
|
||||
raise ValueError(
|
||||
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
|
||||
)
|
||||
if patch.indices.numel() != patch.values.numel():
|
||||
raise ValueError(
|
||||
f"`indices` and `values` must have matching lengths for {patch.name}"
|
||||
)
|
||||
|
||||
stream = args.stream or torch.cuda.current_stream()
|
||||
for patch in iterator:
|
||||
args.group.broadcast(patch.indices, src=args.src, stream=stream)
|
||||
args.group.broadcast(patch.values, src=args.src, stream=stream)
|
||||
def _post_send_sync(self) -> None:
|
||||
"""Wait for the broadcasts to land before returning, so a caller may
|
||||
rebuild or free the patch tensors as soon as `send_weights` returns
|
||||
rather than relying on same-stream ordering. See
|
||||
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
|
||||
cross-rank barrier (and sparse is single-rank on the trainer anyway)."""
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
# Trainer-side process-group setup (shared with the dense engine).
|
||||
trainer_init = staticmethod(trainer_init)
|
||||
def shutdown(self) -> None:
|
||||
self.model_update_group = None
|
||||
|
||||
Reference in New Issue
Block a user