[rl] Stateful Trainer Send: NCCL + Sparse NCCL [3/N] (#50902)
Signed-off-by: haoaaron <ahao@anyscale.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
43d691ec6b
commit
21ea5b4fa1
@@ -2,11 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NCCL-based (dense) weight transfer engine."""
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from collections.abc import Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import torch
|
||||
from typing_extensions import Self
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
@@ -14,72 +16,76 @@ if TYPE_CHECKING:
|
||||
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
ParamMeta,
|
||||
TrainerInitInfo,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
WeightTransferEngine,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
trainer_init,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
trainer_init as open_trainer_endpoint,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
DEFAULT_PACKED_NUM_BUFFERS,
|
||||
packed_nccl_broadcast_consumer,
|
||||
packed_nccl_broadcast_producer,
|
||||
)
|
||||
|
||||
# Re-exported for backward compatibility; canonical home is nccl_common.
|
||||
# NCCLWeightTransferInitInfo is re-exported here for convenience; its canonical
|
||||
# home is nccl_common, shared with the sparse backend.
|
||||
__all__ = [
|
||||
"NCCLWeightTransferInitInfo",
|
||||
"NCCLTrainerSendWeightsArgs",
|
||||
"NCCLTrainerInitInfo",
|
||||
"NCCLWeightTransferUpdateInfo",
|
||||
"NCCLWeightTransferEngine",
|
||||
"NCCLTrainerWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLTrainerSendWeightsArgs:
|
||||
"""Arguments for NCCL trainer_send_weights method."""
|
||||
class NCCLTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for the dense NCCL weight transfer backend.
|
||||
|
||||
group: Any
|
||||
"""Process group (PyNcclCommunicator) for NCCL communication."""
|
||||
src: int = 0
|
||||
"""Source rank (default 0, trainer is typically rank 0)."""
|
||||
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor] | None = None
|
||||
"""Optional function to apply to each (name, tensor) pair before broadcasting.
|
||||
If None, extracts just the tensor."""
|
||||
packed: bool = False
|
||||
"""Whether to use packed tensor broadcasting for efficiency.
|
||||
When True, multiple tensors are batched together before broadcasting
|
||||
to reduce NCCL communication overhead."""
|
||||
stream: torch.cuda.Stream | None = None
|
||||
"""CUDA stream to use for broadcasting if packed is False.
|
||||
If packed is True, new streams will be created for each buffer."""
|
||||
The sender opens its endpoint as NCCL rank 0, so it needs no `rank_offset`.
|
||||
`world_size` is the full trainer+worker NCCL group size. `rank` (from
|
||||
`TrainerInitInfo`) identifies this trainer process; rank 0 is the sender.
|
||||
|
||||
`packed` / buffer sizes are the transfer's wire params. The trainer
|
||||
propagates them to the worker at `trainer_init` so the two sides cannot
|
||||
disagree. Note this defaults to packed, unlike the worker-side
|
||||
`NCCLWeightTransferInitInfo`, whose default only applies when no trainer
|
||||
ships a value. `backend` is the factory dispatch key."""
|
||||
|
||||
backend: ClassVar[str] = "nccl"
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
packed: bool = True
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
"""Size in bytes for each packed tensor buffer.
|
||||
Must match the value used in NCCLWeightTransferUpdateInfo."""
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
|
||||
"""Number of buffers for double/triple buffering during packed transfer.
|
||||
Must match the value used in NCCLWeightTransferUpdateInfo."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Update info for NCCL weight transfer backend."""
|
||||
"""Per-round update info for the dense NCCL weight transfer backend.
|
||||
|
||||
Whether the transfer is packed (and the buffer geometry) is a must-agree
|
||||
wire param carried on the init info (`NCCLTrainerInitInfo` /
|
||||
`NCCLWeightTransferInitInfo`), not here; this carries only the per-round
|
||||
parameter metadata.
|
||||
"""
|
||||
|
||||
names: list[str]
|
||||
dtype_names: list[str]
|
||||
shapes: list[list[int]]
|
||||
packed: bool = False
|
||||
"""Whether to use packed tensor broadcasting for efficiency.
|
||||
When True, multiple tensors are batched together before broadcasting
|
||||
to reduce NCCL communication overhead."""
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
"""Size in bytes for each packed tensor buffer.
|
||||
Both producer and consumer must use the same value."""
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS
|
||||
"""Number of buffers for double/triple buffering during packed transfer.
|
||||
Both producer and consumer must use the same value."""
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate that all lists have the same length."""
|
||||
@@ -121,15 +127,25 @@ class NCCLWeightTransferEngine(
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
# Set from the trainer-supplied init info at the handshake; defaults are
|
||||
# only for the (unreachable) receive-before-init case.
|
||||
self.packed = False
|
||||
self.packed_buffer_size_bytes = DEFAULT_PACKED_BUFFER_SIZE_BYTES
|
||||
self.packed_num_buffers = DEFAULT_PACKED_NUM_BUFFERS
|
||||
|
||||
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
|
||||
"""
|
||||
Initialize NCCL process group with the trainer.
|
||||
Initialize NCCL process group with the trainer and record the
|
||||
trainer-supplied wire params so the worker decodes exactly as the
|
||||
trainer encodes.
|
||||
|
||||
Args:
|
||||
init_info: NCCL initialization info containing master address, port,
|
||||
rank offset, and world size
|
||||
rank offset, world size, and the packed wire params
|
||||
"""
|
||||
self.packed = init_info.packed
|
||||
self.packed_buffer_size_bytes = init_info.packed_buffer_size_bytes
|
||||
self.packed_num_buffers = init_info.packed_num_buffers
|
||||
self.model_update_group = worker_init_process_group(
|
||||
init_info, self.parallel_config
|
||||
)
|
||||
@@ -154,13 +170,14 @@ class NCCLWeightTransferEngine(
|
||||
"""
|
||||
Receive weights from trainer via NCCL broadcast.
|
||||
|
||||
If update_info.packed is True, uses packed tensor broadcasting for
|
||||
efficient transfer of multiple weights in batches. Otherwise, uses simple
|
||||
one-by-one broadcasting.
|
||||
Whether to use packed broadcasting (and the buffer geometry) is read
|
||||
from `self.packed` / `self.packed_*`, set at the init handshake from the
|
||||
trainer's init info, so it is guaranteed to match how the trainer
|
||||
encoded.
|
||||
|
||||
Args:
|
||||
update_info: NCCL update info containing parameter names, dtypes, shapes,
|
||||
and packed flag
|
||||
update_info: NCCL update info containing parameter names, dtypes,
|
||||
and shapes
|
||||
"""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
@@ -173,7 +190,7 @@ class NCCLWeightTransferEngine(
|
||||
)
|
||||
|
||||
with disable_mtp_completeness_check():
|
||||
if update_info.packed:
|
||||
if self.packed:
|
||||
# Build iterator of (name, (shape, dtype)) from update_info
|
||||
def state_dict_info_iterator():
|
||||
for name, dtype_name, shape in zip(
|
||||
@@ -187,8 +204,8 @@ class NCCLWeightTransferEngine(
|
||||
group=self.model_update_group,
|
||||
src=0,
|
||||
post_unpack_func=self.model.load_weights,
|
||||
buffer_size_bytes=update_info.packed_buffer_size_bytes,
|
||||
num_buffers=update_info.packed_num_buffers,
|
||||
buffer_size_bytes=self.packed_buffer_size_bytes,
|
||||
num_buffers=self.packed_num_buffers,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
@@ -210,63 +227,241 @@ class NCCLWeightTransferEngine(
|
||||
self.model_update_group = None
|
||||
|
||||
@staticmethod
|
||||
def trainer_send_weights(
|
||||
iterator: Iterator[tuple[str, torch.Tensor]],
|
||||
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
|
||||
) -> None:
|
||||
"""Broadcast dense weights from trainer to vLLM workers.
|
||||
def trainer_send_weights(*args: Any, **kwargs: Any) -> None:
|
||||
"""Removed. Use the stateful `NCCLTrainerWeightTransferEngine` instead.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of model parameters. Returns (name, tensor) tuples
|
||||
trainer_args: Dictionary or NCCLTrainerSendWeightsArgs instance containing
|
||||
NCCL-specific arguments. If a dict, should contain keys from
|
||||
NCCLTrainerSendWeightsArgs.
|
||||
|
||||
Example:
|
||||
>>> from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
... NCCLWeightTransferEngine,
|
||||
... NCCLTrainerSendWeightsArgs,
|
||||
... )
|
||||
>>> param_iter = ((n, p) for n, p in model.named_parameters())
|
||||
>>> args = NCCLTrainerSendWeightsArgs(group=group, packed=True)
|
||||
>>> NCCLWeightTransferEngine.trainer_send_weights(param_iter, args)
|
||||
Transitional stub kept only to satisfy the (still abstract)
|
||||
`WeightTransferEngine.trainer_send_weights`; that member is dropped from
|
||||
the worker ABC once every backend has migrated to the trainer engine.
|
||||
"""
|
||||
# Parse trainer args - accept either dict or dataclass instance
|
||||
if isinstance(trainer_args, dict):
|
||||
args = NCCLTrainerSendWeightsArgs(**trainer_args)
|
||||
else:
|
||||
args = trainer_args
|
||||
raise NotImplementedError(
|
||||
"The static NCCL trainer path has been replaced by "
|
||||
"NCCLTrainerWeightTransferEngine. Build it via "
|
||||
"WeightTransferTrainerFactory.trainer_init(NCCLTrainerInitInfo(...), "
|
||||
"client=..., source=...) and drive it with send_weights()."
|
||||
)
|
||||
|
||||
if args.post_iter_func is None:
|
||||
# Default: extract just the tensor from (name, tensor) tuple
|
||||
post_iter_func = lambda x: x[1]
|
||||
else:
|
||||
post_iter_func = args.post_iter_func
|
||||
|
||||
if args.packed:
|
||||
# Use packed tensor broadcasting for efficiency
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
packed_nccl_broadcast_producer,
|
||||
class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerInitInfo]):
|
||||
"""Trainer-side NCCL weight transfer engine.
|
||||
|
||||
On the sender (rank 0) holds the NCCL communicator and drives the full
|
||||
update round trip: it runs the inference-side `update_weights` concurrently
|
||||
with the trainer-side broadcast (both rendezvous inside the same NCCL
|
||||
calls), then finishes the update. Non-sender trainer ranks hold no
|
||||
communicator; they only iterate the source to stay in the trainer-side
|
||||
collective (e.g. FSDP `full_tensor()`) and skip the client RPCs and the
|
||||
broadcast (all guarded on `is_sender`).
|
||||
|
||||
`packed` / buffer sizes come from `NCCLTrainerInitInfo`; the sender
|
||||
propagates them to the worker at `trainer_init` (on the worker-side init
|
||||
info), so per-round payloads carry only parameter metadata.
|
||||
"""
|
||||
|
||||
init_info_cls = NCCLTrainerInitInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource,
|
||||
is_sender: bool = True,
|
||||
packed: bool = True,
|
||||
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
packed_num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
|
||||
) -> None:
|
||||
super().__init__(client=client, source=source, is_sender=is_sender)
|
||||
self.packed = packed
|
||||
self.packed_buffer_size_bytes = packed_buffer_size_bytes
|
||||
self.packed_num_buffers = packed_num_buffers
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
init_info: NCCLTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
if source is None:
|
||||
raise ValueError("NCCL trainer weight transfer requires a WeightSource.")
|
||||
engine = cls(
|
||||
client=client,
|
||||
source=source,
|
||||
is_sender=init_info.is_sender,
|
||||
packed=init_info.packed,
|
||||
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
|
||||
packed_num_buffers=init_info.packed_num_buffers,
|
||||
)
|
||||
if not engine.is_sender:
|
||||
# Non-sender trainer ranks aren't part of the transfer NCCL group and
|
||||
# don't drive the inference side; they only participate in the
|
||||
# trainer-side gather during send_weights.
|
||||
return engine
|
||||
|
||||
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
|
||||
worker_init_info = NCCLWeightTransferInitInfo(
|
||||
master_address=init_info.master_address,
|
||||
master_port=init_info.master_port,
|
||||
rank_offset=1,
|
||||
world_size=init_info.world_size,
|
||||
packed=init_info.packed,
|
||||
packed_buffer_size_bytes=init_info.packed_buffer_size_bytes,
|
||||
packed_num_buffers=init_info.packed_num_buffers,
|
||||
)
|
||||
|
||||
# The inference workers block inside init_weight_transfer_engine waiting
|
||||
# for the NCCL rendezvous, so we kick that off on a side thread while we
|
||||
# open the trainer endpoint (rank 0); both sides must rendezvous together.
|
||||
with ThreadPoolExecutor(max_workers=1) as exe:
|
||||
future = exe.submit(
|
||||
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
|
||||
)
|
||||
engine.model_update_group = open_trainer_endpoint(init_info)
|
||||
future.result() # surface any inference-side init error
|
||||
|
||||
return engine
|
||||
|
||||
def send_weights(self) -> None:
|
||||
assert self.source is not None # guaranteed by trainer_init / __init__
|
||||
source = self.source
|
||||
|
||||
# Metadata is declared without gathering. For Megatron it is itself a
|
||||
# collective, so every rank runs it; only the sender ships it.
|
||||
meta = source.metadata()
|
||||
|
||||
if not self.is_sender:
|
||||
# Non-sender ranks only join the trainer-side gather collective.
|
||||
self._broadcast(source, meta)
|
||||
self._post_send_sync()
|
||||
return
|
||||
|
||||
update_info = NCCLWeightTransferUpdateInfo(
|
||||
names=[m.name for m in meta],
|
||||
dtype_names=[str(m.dtype).split(".")[-1] for m in meta],
|
||||
shapes=[list(m.shape) for m in meta],
|
||||
)
|
||||
|
||||
self.client.start_weight_update()
|
||||
# update_weights (workers receive) must run concurrently with the
|
||||
# trainer-side broadcast — both rendezvous inside the same NCCL calls.
|
||||
exe = ThreadPoolExecutor(max_workers=1)
|
||||
try:
|
||||
future = exe.submit(self.client.update_weights, asdict(update_info))
|
||||
# Cheap best-effort: if update_weights already failed (e.g. a bad
|
||||
# request rejected before any NCCL call), surface it now instead of
|
||||
# hanging in broadcast waiting for a peer that will never arrive.
|
||||
if future.done():
|
||||
future.result()
|
||||
self._broadcast(source, meta)
|
||||
future.result() # surface inference-side errors
|
||||
finally:
|
||||
# Never wait for the RPC thread here. If the broadcast raised, the
|
||||
# worker is still blocked in the matching NCCL call and will never
|
||||
# return, so joining would turn the error into a permanent hang.
|
||||
# Let the exception out instead; the transfer group is unusable
|
||||
# either way and the caller has to tear it down. (The orphaned
|
||||
# thread is still joined at interpreter exit, so a caller that wants
|
||||
# to exit cleanly after such a failure must not wait on it.)
|
||||
exe.shutdown(wait=False)
|
||||
self.client.finish_weight_update()
|
||||
self._post_send_sync()
|
||||
|
||||
def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None:
|
||||
"""Iterate the source (materializing each tensor — a collective on all
|
||||
ranks) and, on the sender, broadcast from rank 0, packed or one-by-one.
|
||||
Non-sender ranks only replay the iteration to stay in the collective."""
|
||||
if not self.is_sender:
|
||||
for _ in source:
|
||||
pass
|
||||
return
|
||||
|
||||
assert self.model_update_group is not None, (
|
||||
"trainer_init() must be called before _broadcast()."
|
||||
)
|
||||
pairs = self._checked_iter(source, meta)
|
||||
if self.packed:
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=iterator,
|
||||
group=args.group,
|
||||
src=args.src,
|
||||
post_iter_func=post_iter_func,
|
||||
buffer_size_bytes=args.packed_buffer_size_bytes,
|
||||
num_buffers=args.packed_num_buffers,
|
||||
iterator=pairs,
|
||||
group=self.model_update_group,
|
||||
src=0,
|
||||
post_iter_func=lambda item: item[1],
|
||||
buffer_size_bytes=self.packed_buffer_size_bytes,
|
||||
num_buffers=self.packed_num_buffers,
|
||||
)
|
||||
else:
|
||||
# Use simple one-by-one broadcasting
|
||||
for item in iterator:
|
||||
tensor = post_iter_func(item)
|
||||
args.group.broadcast(
|
||||
tensor,
|
||||
src=args.src,
|
||||
stream=args.stream or torch.cuda.current_stream(),
|
||||
)
|
||||
stream = torch.cuda.current_stream()
|
||||
for _name, tensor in pairs:
|
||||
# NCCL sends `numel` elements straight from `data_ptr()`, so a
|
||||
# non-contiguous view would ship whatever follows its base
|
||||
# pointer. Keep the copy referenced until the broadcast is
|
||||
# enqueued. (The packed path linearizes in `pack_tensors`.)
|
||||
send = tensor if tensor.is_contiguous() else tensor.contiguous()
|
||||
self.model_update_group.broadcast(send, src=0, stream=stream)
|
||||
|
||||
# Trainer-side process-group setup. Delegates to the shared helper so the
|
||||
# sparse engine can reuse the exact same rendezvous without subclassing.
|
||||
trainer_init = staticmethod(trainer_init)
|
||||
@staticmethod
|
||||
def _checked_iter(
|
||||
source: WeightSource, meta: list[ParamMeta]
|
||||
) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""Yield the source's pairs, checking each against what the worker was
|
||||
told to expect.
|
||||
|
||||
The worker sizes its receive buffers — and in packed mode cuts its chunk
|
||||
boundaries — from the update info, which is built from `metadata()`. If
|
||||
iteration disagrees with it, the two sides split the stream differently
|
||||
and the transfer hangs in NCCL or loads garbage. Checking here costs one
|
||||
comparison per parameter and turns that into an error naming the first
|
||||
divergent parameter. Sender-only: under pipeline parallelism a
|
||||
non-sender's yielded tensor is not meaningful.
|
||||
"""
|
||||
sent = 0
|
||||
for name, tensor in source:
|
||||
if sent >= len(meta):
|
||||
raise ValueError(
|
||||
f"WeightSource yielded more parameters than metadata() "
|
||||
f"declared ({len(meta)}); first extra is {name!r}."
|
||||
)
|
||||
expected = meta[sent]
|
||||
if (
|
||||
name != expected.name
|
||||
or tensor.dtype != expected.dtype
|
||||
or tuple(tensor.shape) != expected.shape
|
||||
):
|
||||
raise ValueError(
|
||||
"WeightSource metadata() disagrees with iteration at index "
|
||||
f"{sent}: declared {expected.name!r} "
|
||||
f"{expected.dtype} {tuple(expected.shape)}, got {name!r} "
|
||||
f"{tensor.dtype} {tuple(tensor.shape)}. Both channels must "
|
||||
"enumerate the same parameters in the same order."
|
||||
)
|
||||
sent += 1
|
||||
yield name, tensor
|
||||
if sent != len(meta):
|
||||
raise ValueError(
|
||||
f"WeightSource yielded {sent} parameters but metadata() "
|
||||
f"declared {len(meta)}; the worker is waiting for the rest."
|
||||
)
|
||||
|
||||
def _post_send_sync(self) -> None:
|
||||
"""Wait for this rank's transfer work to land before returning.
|
||||
|
||||
Broadcasts are only *enqueued* by `send_weights`: the unpacked path on
|
||||
the current stream, the packed path on the producer's own streams (which
|
||||
it drains itself). Waiting here lets a caller mutate parameters, or
|
||||
start the next step on another stream, as soon as `send_weights`
|
||||
returns, instead of silently depending on same-stream ordering. Every
|
||||
rank waits: a non-sender's `full_tensor()` gathers feed the sender's
|
||||
broadcast, so they must have landed before it may touch its shards.
|
||||
|
||||
Unlike IPC there is no cross-rank barrier here. Nothing in this backend
|
||||
outlives the collective it travelled in (IPC's barrier keeps shared
|
||||
buffers alive until every consumer has opened them), so a barrier would
|
||||
only add a dependency on the default process group that this backend
|
||||
otherwise does not have.
|
||||
"""
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.model_update_group = None
|
||||
|
||||
Reference in New Issue
Block a user