[rl] Stateful Trainer Send: NCCL + Sparse NCCL [3/N] (#50902)
Signed-off-by: haoaaron <ahao@anyscale.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
43d691ec6b
commit
21ea5b4fa1
@@ -14,11 +14,13 @@ MVP limitations:
|
||||
* not composable with checkpoint-format or packed updates
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from collections.abc import Iterable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import torch
|
||||
from typing_extensions import Self
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
@@ -26,20 +28,27 @@ if TYPE_CHECKING:
|
||||
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
TrainerInitInfo,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
WeightTransferEngine,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
trainer_init,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerSendWeightsArgs
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
trainer_init as open_trainer_endpoint,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SparseWeightPatch",
|
||||
"SparseNCCLTrainerInitInfo",
|
||||
"SparseNCCLWeightTransferUpdateInfo",
|
||||
"SparseNCCLWeightTransferEngine",
|
||||
"SparseNCCLTrainerWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
@@ -50,6 +59,25 @@ class SparseWeightPatch:
|
||||
name: str
|
||||
indices: torch.Tensor
|
||||
values: torch.Tensor
|
||||
full_shape: tuple[int, ...] | None = None
|
||||
"""Full shape of the patched parameter. Required when the patch is sent
|
||||
via `SparseNCCLTrainerWeightTransferEngine` (it ships in the per-round
|
||||
update info); the worker-side apply path does not read it."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseNCCLTrainerInitInfo(TrainerInitInfo):
|
||||
"""Trainer-side init info for the sparse NCCL weight transfer backend.
|
||||
|
||||
Same rendezvous shape as the dense NCCL backend (the sender opens its
|
||||
endpoint as NCCL rank 0), but with no packed wire params: sparse transfers
|
||||
are never packed. `backend` is the factory dispatch key."""
|
||||
|
||||
backend: ClassVar[str] = "sparse_nccl"
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
world_size: int
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -200,25 +228,182 @@ class SparseNCCLWeightTransferEngine(
|
||||
self.model_update_group = None
|
||||
|
||||
@staticmethod
|
||||
def trainer_send_weights(
|
||||
iterator: Iterator[SparseWeightPatch],
|
||||
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
|
||||
) -> None:
|
||||
"""Broadcast sparse flat-index patches from trainer to vLLM workers."""
|
||||
if isinstance(trainer_args, dict):
|
||||
args = NCCLTrainerSendWeightsArgs(**trainer_args)
|
||||
else:
|
||||
args = trainer_args
|
||||
def trainer_send_weights(*args: Any, **kwargs: Any) -> None:
|
||||
"""Removed. Use the stateful `SparseNCCLTrainerWeightTransferEngine`.
|
||||
|
||||
if args.packed:
|
||||
Transitional stub kept only to satisfy the (still abstract)
|
||||
`WeightTransferEngine.trainer_send_weights`; that member is dropped from
|
||||
the worker ABC once every backend has migrated to the trainer engine.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"The static sparse NCCL trainer path has been replaced by "
|
||||
"SparseNCCLTrainerWeightTransferEngine. Build it via "
|
||||
"WeightTransferTrainerFactory.trainer_init("
|
||||
"SparseNCCLTrainerInitInfo(...), client=..., source=...) and drive "
|
||||
"it with send_weights()."
|
||||
)
|
||||
|
||||
|
||||
class SparseNCCLTrainerWeightTransferEngine(
|
||||
TrainerWeightTransferEngine[SparseNCCLTrainerInitInfo]
|
||||
):
|
||||
"""Trainer-side sparse NCCL weight transfer engine.
|
||||
|
||||
Broadcasts flat-index (indices, values) patches from NCCL rank 0 while the
|
||||
inference-side `update_weights` runs concurrently on a side thread (the
|
||||
worker's recvs rendezvous inside the same NCCL broadcasts), then finishes
|
||||
the update. Unlike the full-resync backends, sparse patches differ every
|
||||
round (a fresh set of deltas from each optimizer step), so they are not a
|
||||
stable `WeightSource`: the engine takes no `source`, and each round's
|
||||
patches are passed straight to `send_weights(patches)`. A round with no
|
||||
patches is a no-op.
|
||||
|
||||
The sparse backend assumes a single-rank trainer (matching its TP=1 / PP=1
|
||||
MVP scope), so non-sender ranks skip `send_weights` entirely.
|
||||
"""
|
||||
|
||||
init_info_cls = SparseNCCLTrainerInitInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
is_sender: bool = True,
|
||||
) -> None:
|
||||
# Sparse is a delta backend: it takes per-round patches via
|
||||
# send_weights, so a `source` would silently never be sent. The
|
||||
# parameter exists only to match the base/factory signature.
|
||||
if source is not None:
|
||||
raise ValueError(
|
||||
"Sparse NCCL updates cannot be combined with `packed=True`"
|
||||
"Sparse NCCL weight transfer takes no WeightSource; pass each "
|
||||
"round's patches to send_weights(patches) instead."
|
||||
)
|
||||
super().__init__(client=client, source=source, is_sender=is_sender)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
@classmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
init_info: SparseNCCLTrainerInitInfo,
|
||||
*,
|
||||
client: VLLMWeightSyncClient,
|
||||
source: WeightSource | None = None,
|
||||
) -> Self:
|
||||
engine = cls(client=client, source=source, is_sender=init_info.is_sender)
|
||||
if not engine.is_sender:
|
||||
return engine
|
||||
|
||||
# Workers sit at rank_offset 1, after the single trainer sender rank 0.
|
||||
# Sparse transfers are never packed, so the worker keeps the unpacked
|
||||
# defaults on its init info.
|
||||
worker_init_info = NCCLWeightTransferInitInfo(
|
||||
master_address=init_info.master_address,
|
||||
master_port=init_info.master_port,
|
||||
rank_offset=1,
|
||||
world_size=init_info.world_size,
|
||||
)
|
||||
|
||||
# The inference workers block inside init_weight_transfer_engine waiting
|
||||
# for the NCCL rendezvous, so we kick that off on a side thread while we
|
||||
# open the trainer endpoint (rank 0); both sides must rendezvous together.
|
||||
with ThreadPoolExecutor(max_workers=1) as exe:
|
||||
future = exe.submit(
|
||||
engine.client.init_weight_transfer_engine, asdict(worker_init_info)
|
||||
)
|
||||
# Open the trainer endpoint as NCCL rank 0 on the current device
|
||||
# (the init info satisfies the helper's rendezvous protocol).
|
||||
engine.model_update_group = open_trainer_endpoint(init_info)
|
||||
future.result() # surface any inference-side init error
|
||||
|
||||
return engine
|
||||
|
||||
def send_weights(self, patches: Iterable[SparseWeightPatch] | None = None) -> None:
|
||||
"""Broadcast this round's sparse patches. `patches` is the per-round
|
||||
payload (sparse deltas differ every round), so it is passed here rather
|
||||
than fixed at init. Every patch must set `full_shape`."""
|
||||
if not self.is_sender:
|
||||
return
|
||||
|
||||
patches = list(patches) if patches is not None else []
|
||||
if not patches:
|
||||
return
|
||||
|
||||
shapes = []
|
||||
for patch in patches:
|
||||
if patch.full_shape is None:
|
||||
raise ValueError(
|
||||
"SparseWeightPatch.full_shape must be set to send via the "
|
||||
f"trainer engine: {patch.name}"
|
||||
)
|
||||
self._validate_patch(patch)
|
||||
shapes.append(list(patch.full_shape))
|
||||
|
||||
update_info = SparseNCCLWeightTransferUpdateInfo(
|
||||
names=[patch.name for patch in patches],
|
||||
dtype_names=[str(patch.values.dtype).split(".")[-1] for patch in patches],
|
||||
shapes=shapes,
|
||||
num_updates_list=[patch.indices.numel() for patch in patches],
|
||||
)
|
||||
|
||||
assert self.model_update_group is not None, (
|
||||
"trainer_init() must be called before send_weights()."
|
||||
)
|
||||
self.client.start_weight_update()
|
||||
# update_weights (workers receive) must run concurrently with the
|
||||
# trainer-side broadcasts — both rendezvous inside the same NCCL calls.
|
||||
exe = ThreadPoolExecutor(max_workers=1)
|
||||
try:
|
||||
future = exe.submit(self.client.update_weights, asdict(update_info))
|
||||
# Cheap best-effort: if update_weights already failed (e.g. a bad
|
||||
# request rejected before any NCCL call), surface it now instead of
|
||||
# hanging in broadcast waiting for a peer that will never arrive.
|
||||
if future.done():
|
||||
future.result()
|
||||
stream = torch.cuda.current_stream()
|
||||
for patch in patches:
|
||||
self.model_update_group.broadcast(patch.indices, src=0, stream=stream)
|
||||
self.model_update_group.broadcast(patch.values, src=0, stream=stream)
|
||||
future.result() # surface inference-side errors
|
||||
finally:
|
||||
# Never wait for the RPC thread here: if a broadcast raised, the
|
||||
# worker is still blocked in the matching NCCL call and will never
|
||||
# return, so joining would turn the error into a permanent hang.
|
||||
# See NCCLTrainerWeightTransferEngine.send_weights.
|
||||
exe.shutdown(wait=False)
|
||||
self.client.finish_weight_update()
|
||||
self._post_send_sync()
|
||||
|
||||
@staticmethod
|
||||
def _validate_patch(patch: SparseWeightPatch) -> None:
|
||||
"""Reject a malformed patch before any NCCL call.
|
||||
|
||||
The worker checks the same invariants in `_apply_patch`, but by then the
|
||||
broadcasts are already under way: a mismatch surfaces as a size mismatch
|
||||
mid-transfer, which wedges both sides instead of raising. Checking here
|
||||
keeps the failure on the trainer, before `start_weight_update`.
|
||||
"""
|
||||
if patch.indices.dtype != torch.int32:
|
||||
raise ValueError(
|
||||
f"Sparse weight updates require int32 indices: {patch.name}"
|
||||
)
|
||||
if patch.indices.ndim != 1 or patch.values.ndim != 1:
|
||||
raise ValueError(
|
||||
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
|
||||
)
|
||||
if patch.indices.numel() != patch.values.numel():
|
||||
raise ValueError(
|
||||
f"`indices` and `values` must have matching lengths for {patch.name}"
|
||||
)
|
||||
|
||||
stream = args.stream or torch.cuda.current_stream()
|
||||
for patch in iterator:
|
||||
args.group.broadcast(patch.indices, src=args.src, stream=stream)
|
||||
args.group.broadcast(patch.values, src=args.src, stream=stream)
|
||||
def _post_send_sync(self) -> None:
|
||||
"""Wait for the broadcasts to land before returning, so a caller may
|
||||
rebuild or free the patch tensors as soon as `send_weights` returns
|
||||
rather than relying on same-stream ordering. See
|
||||
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
|
||||
cross-rank barrier (and sparse is single-rank on the trainer anyway)."""
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
# Trainer-side process-group setup (shared with the dense engine).
|
||||
trainer_init = staticmethod(trainer_init)
|
||||
def shutdown(self) -> None:
|
||||
self.model_update_group = None
|
||||
|
||||
Reference in New Issue
Block a user