[RL] Support checkpoint-coordinate sparse NCCL weight updates (#53751)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
aoshen02
OpenAI Codex
parent
50708f95ae
commit
6c4aa8bec3
@@ -2,20 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sparse NCCL weight transfer engine.
|
||||
|
||||
A standalone engine (not a subclass of `NCCLWeightTransferEngine`) for applying
|
||||
sparse, flat-index weight patches in place. It shares only NCCL process-group
|
||||
initialization with the dense engine (via `nccl_common`); the update path
|
||||
applies index/value patches directly to existing model parameters and never runs
|
||||
layerwise reload.
|
||||
|
||||
MVP limitations:
|
||||
* TP=1 and PP=1 only
|
||||
* uses runtime/kernel-format parameter names
|
||||
* not composable with checkpoint-format or packed updates
|
||||
Sparse patches use checkpoint names, shapes, and flat indices. The model's native
|
||||
weight loader maps them to rank-local runtime parameters, including TP shards and
|
||||
packed parameters.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
@@ -42,6 +36,10 @@ from vllm.distributed.weight_transfer.nccl_common import (
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
trainer_init as open_trainer_endpoint,
|
||||
)
|
||||
from vllm.model_executor.model_loader.checkpoint_weight_patch import (
|
||||
CheckpointWeightPatch,
|
||||
load_checkpoint_weight_patches,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SparseWeightPatch",
|
||||
@@ -54,15 +52,13 @@ __all__ = [
|
||||
|
||||
@dataclass
|
||||
class SparseWeightPatch:
|
||||
"""A sparse in-place patch for one existing parameter."""
|
||||
"""A sparse patch in checkpoint coordinates."""
|
||||
|
||||
name: str
|
||||
indices: torch.Tensor
|
||||
values: torch.Tensor
|
||||
full_shape: tuple[int, ...] | None = None
|
||||
"""Full shape of the patched parameter. Required when the patch is sent
|
||||
via `SparseNCCLTrainerWeightTransferEngine` (it ships in the per-round
|
||||
update info); the worker-side apply path does not read it."""
|
||||
full_shape: tuple[int, ...]
|
||||
"""Full checkpoint shape."""
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -119,10 +115,9 @@ class SparseNCCLWeightTransferEngine(
|
||||
"""
|
||||
Sparse weight transfer engine using NCCL.
|
||||
|
||||
Receives flat-index (indices, values) patches broadcast from the trainer
|
||||
(rank 0) and applies them in place to existing model parameters. Weights are
|
||||
applied directly without layerwise reload, so `start_weight_update` and
|
||||
`finish_weight_update` are no-ops.
|
||||
Receives checkpoint-coordinate patches broadcast from the trainer and applies
|
||||
them through the model's native weight loader. Sparse updates modify initialized
|
||||
model tensors in place, so the layerwise reload lifecycle is not used.
|
||||
"""
|
||||
|
||||
init_info_cls = NCCLWeightTransferInitInfo
|
||||
@@ -147,10 +142,7 @@ class SparseNCCLWeightTransferEngine(
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""No-op: sparse patches are applied in place, no layerwise reload."""
|
||||
if self.parallel_config.world_size != 1:
|
||||
raise NotImplementedError(
|
||||
"Sparse weight updates currently require TP=1 and PP=1"
|
||||
)
|
||||
pass
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""No-op: sparse patches are applied in place, no layerwise reload."""
|
||||
@@ -164,64 +156,34 @@ class SparseNCCLWeightTransferEngine(
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
|
||||
# Use the worker's assigned device rather than the ambient current
|
||||
# device: the receive path is no longer wrapped in
|
||||
# `with torch.device(self.device)` by the caller, so the current device
|
||||
# is not guaranteed to match self.device. The recv buffers must live on
|
||||
# the same device as the NCCL communicator (created on self.device).
|
||||
device = self.device
|
||||
for name, dtype_name, num_updates in zip(
|
||||
update_info.names,
|
||||
update_info.dtype_names,
|
||||
update_info.num_updates_list,
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
|
||||
values = torch.empty(num_updates, dtype=dtype, device=device)
|
||||
self.model_update_group.broadcast(
|
||||
indices, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
self.model_update_group.broadcast(
|
||||
values, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
self._apply_patch(
|
||||
SparseWeightPatch(name=name, indices=indices, values=values)
|
||||
)
|
||||
del indices
|
||||
del values
|
||||
|
||||
def _apply_patch(self, patch: SparseWeightPatch) -> None:
|
||||
"""Apply a single sparse flat-index patch to an existing model param."""
|
||||
param = self.model.get_parameter(patch.name)
|
||||
if not param.data.is_contiguous():
|
||||
raise NotImplementedError(
|
||||
"Sparse weight updates currently require contiguous params: "
|
||||
f"{patch.name}"
|
||||
)
|
||||
if patch.indices.dtype != torch.int32:
|
||||
raise ValueError(
|
||||
f"Sparse weight updates currently require int32 indices: {patch.name}"
|
||||
)
|
||||
if patch.indices.ndim != 1 or patch.values.ndim != 1:
|
||||
raise ValueError(
|
||||
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
|
||||
)
|
||||
if patch.indices.numel() != patch.values.numel():
|
||||
raise ValueError(
|
||||
f"`indices` and `values` must have matching lengths for {patch.name}"
|
||||
)
|
||||
if patch.values.dtype != param.dtype:
|
||||
raise ValueError(
|
||||
f"Sparse values dtype {patch.values.dtype} does not match "
|
||||
f"parameter dtype {param.dtype} for {patch.name}"
|
||||
)
|
||||
|
||||
flat_param = param.data.view(-1)
|
||||
flat_param.index_copy_(
|
||||
0,
|
||||
patch.indices.to(device=flat_param.device, dtype=torch.long),
|
||||
patch.values.to(device=flat_param.device),
|
||||
)
|
||||
# Receive on the communicator's device rather than relying on the
|
||||
# ambient current device in the RPC thread.
|
||||
device = self.model_update_group.device
|
||||
with torch.accelerator.device_index(device.index):
|
||||
stream = torch.cuda.current_stream(device=device)
|
||||
patches = []
|
||||
for name, dtype_name, shape, num_updates in zip(
|
||||
update_info.names,
|
||||
update_info.dtype_names,
|
||||
update_info.shapes,
|
||||
update_info.num_updates_list,
|
||||
strict=True,
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
|
||||
values = torch.empty(num_updates, dtype=dtype, device=device)
|
||||
self.model_update_group.broadcast(indices, src=0, stream=stream)
|
||||
self.model_update_group.broadcast(values, src=0, stream=stream)
|
||||
patches.append(
|
||||
CheckpointWeightPatch(
|
||||
name=name,
|
||||
shape=tuple(shape),
|
||||
dtype=dtype,
|
||||
values=values,
|
||||
indices=indices,
|
||||
)
|
||||
)
|
||||
load_checkpoint_weight_patches(self.model, patches)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
@@ -235,15 +197,16 @@ class SparseNCCLTrainerWeightTransferEngine(
|
||||
|
||||
Broadcasts flat-index (indices, values) patches from NCCL rank 0 while the
|
||||
inference-side `update_weights` runs concurrently on a side thread (the
|
||||
worker's recvs rendezvous inside the same NCCL broadcasts), then finishes
|
||||
the update. Unlike the full-resync backends, sparse patches differ every
|
||||
round (a fresh set of deltas from each optimizer step), so they are not a
|
||||
stable `WeightSource`: the engine takes no `source`, and each round's
|
||||
patches are passed straight to `send_weights(patches)`. A round with no
|
||||
patches is a no-op.
|
||||
worker's recvs rendezvous inside the same NCCL broadcasts). `send_weights`
|
||||
owns a complete one-shot lifecycle. RL infrastructure that owns the generic
|
||||
client lifecycle can call `send_weight_chunk` between one start and finish.
|
||||
|
||||
The sparse backend assumes a single-rank trainer (matching its TP=1 / PP=1
|
||||
MVP scope), so non-sender ranks skip `send_weights` entirely.
|
||||
Sparse patches differ every round, so they are not a stable `WeightSource`:
|
||||
the engine takes no `source`, and patches are passed directly to the send
|
||||
methods. An empty patch list is a no-op.
|
||||
|
||||
Only the designated trainer sender joins the transfer group; other trainer
|
||||
ranks skip sparse sends.
|
||||
"""
|
||||
|
||||
init_info_cls = SparseNCCLTrainerInitInfo
|
||||
@@ -303,70 +266,88 @@ class SparseNCCLTrainerWeightTransferEngine(
|
||||
return engine
|
||||
|
||||
def send_weights(self, patches: Iterable[SparseWeightPatch] | None = None) -> None:
|
||||
"""Broadcast this round's sparse patches. `patches` is the per-round
|
||||
payload (sparse deltas differ every round), so it is passed here rather
|
||||
than fixed at init. Every patch must set `full_shape`."""
|
||||
if not self.is_sender:
|
||||
return
|
||||
|
||||
patches = list(patches) if patches is not None else []
|
||||
"""Broadcast one sparse update through a one-shot lifecycle."""
|
||||
patches = self._prepare_patches(patches)
|
||||
if not patches:
|
||||
return
|
||||
|
||||
shapes = []
|
||||
for patch in patches:
|
||||
if patch.full_shape is None:
|
||||
raise ValueError(
|
||||
"SparseWeightPatch.full_shape must be set to send via the "
|
||||
f"trainer engine: {patch.name}"
|
||||
)
|
||||
self._validate_patch(patch)
|
||||
shapes.append(list(patch.full_shape))
|
||||
|
||||
update_info = SparseNCCLWeightTransferUpdateInfo(
|
||||
names=[patch.name for patch in patches],
|
||||
dtype_names=[str(patch.values.dtype).split(".")[-1] for patch in patches],
|
||||
shapes=shapes,
|
||||
num_updates_list=[patch.indices.numel() for patch in patches],
|
||||
)
|
||||
|
||||
assert self.model_update_group is not None, (
|
||||
"trainer_init() must be called before send_weights()."
|
||||
)
|
||||
self.client.start_weight_update()
|
||||
# update_weights (workers receive) must run concurrently with the
|
||||
# trainer-side broadcasts — both rendezvous inside the same NCCL calls.
|
||||
exe = ThreadPoolExecutor(max_workers=1)
|
||||
try:
|
||||
future = exe.submit(self.client.update_weights, asdict(update_info))
|
||||
# Cheap best-effort: if update_weights already failed (e.g. a bad
|
||||
# request rejected before any NCCL call), surface it now instead of
|
||||
# hanging in broadcast waiting for a peer that will never arrive.
|
||||
if future.done():
|
||||
future.result()
|
||||
stream = torch.cuda.current_stream()
|
||||
for patch in patches:
|
||||
self.model_update_group.broadcast(patch.indices, src=0, stream=stream)
|
||||
self.model_update_group.broadcast(patch.values, src=0, stream=stream)
|
||||
future.result() # surface inference-side errors
|
||||
finally:
|
||||
# Never wait for the RPC thread here: if a broadcast raised, the
|
||||
# worker is still blocked in the matching NCCL call and will never
|
||||
# return, so joining would turn the error into a permanent hang.
|
||||
# See NCCLTrainerWeightTransferEngine.send_weights.
|
||||
exe.shutdown(wait=False)
|
||||
self._broadcast_chunk(patches)
|
||||
self.client.finish_weight_update()
|
||||
self._post_send_sync()
|
||||
|
||||
def send_weight_chunk(
|
||||
self, patches: Iterable[SparseWeightPatch] | None = None
|
||||
) -> None:
|
||||
"""Broadcast one chunk inside a caller-owned weight update lifecycle."""
|
||||
patches = self._prepare_patches(patches)
|
||||
if not patches:
|
||||
return
|
||||
self._broadcast_chunk(patches)
|
||||
|
||||
def _prepare_patches(
|
||||
self, patches: Iterable[SparseWeightPatch] | None
|
||||
) -> list[SparseWeightPatch]:
|
||||
if not self.is_sender:
|
||||
return []
|
||||
|
||||
prepared = list(patches) if patches is not None else []
|
||||
if not prepared:
|
||||
return []
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError("trainer_init() must be called before sending weights.")
|
||||
for patch in prepared:
|
||||
self._validate_patch(patch)
|
||||
return prepared
|
||||
|
||||
def _broadcast_chunk(self, patches: list[SparseWeightPatch]) -> None:
|
||||
assert self.model_update_group is not None
|
||||
device = self.model_update_group.device
|
||||
device_context = (
|
||||
torch.accelerator.device_index(device.index)
|
||||
if device.type != "cpu"
|
||||
else nullcontext()
|
||||
)
|
||||
with device_context:
|
||||
update_info = SparseNCCLWeightTransferUpdateInfo(
|
||||
names=[patch.name for patch in patches],
|
||||
dtype_names=[
|
||||
str(patch.values.dtype).split(".")[-1] for patch in patches
|
||||
],
|
||||
shapes=[list(patch.full_shape) for patch in patches],
|
||||
num_updates_list=[patch.indices.numel() for patch in patches],
|
||||
)
|
||||
|
||||
# update_weights (workers receive) must run concurrently with the
|
||||
# trainer-side broadcasts — both rendezvous inside the same NCCL calls.
|
||||
executor = ThreadPoolExecutor(max_workers=1)
|
||||
try:
|
||||
future = executor.submit(
|
||||
self.client.update_weights, asdict(update_info)
|
||||
)
|
||||
# Surface an RPC that failed before the worker entered NCCL rather
|
||||
# than broadcasting to a peer that will never receive.
|
||||
if future.done():
|
||||
future.result()
|
||||
stream = torch.cuda.current_stream()
|
||||
for patch in patches:
|
||||
self.model_update_group.broadcast(
|
||||
patch.indices, src=0, stream=stream
|
||||
)
|
||||
self.model_update_group.broadcast(
|
||||
patch.values, src=0, stream=stream
|
||||
)
|
||||
future.result() # surface inference-side errors
|
||||
finally:
|
||||
# A failed broadcast can leave the RPC blocked in the matching
|
||||
# receive. Waiting for that thread would hide the original error.
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
self._post_send_sync()
|
||||
|
||||
@staticmethod
|
||||
def _validate_patch(patch: SparseWeightPatch) -> None:
|
||||
"""Reject a malformed patch before any NCCL call.
|
||||
|
||||
The worker checks the same invariants in `_apply_patch`, but by then the
|
||||
broadcasts are already under way: a mismatch surfaces as a size mismatch
|
||||
mid-transfer, which wedges both sides instead of raising. Checking here
|
||||
keeps the failure on the trainer, before `start_weight_update`.
|
||||
"""
|
||||
"""Reject a malformed patch before starting the NCCL transfer."""
|
||||
if patch.full_shape is None:
|
||||
raise ValueError(f"Sparse patch requires full_shape: {patch.name}")
|
||||
if patch.indices.dtype != torch.int32:
|
||||
raise ValueError(
|
||||
f"Sparse weight updates require int32 indices: {patch.name}"
|
||||
@@ -382,10 +363,10 @@ class SparseNCCLTrainerWeightTransferEngine(
|
||||
|
||||
def _post_send_sync(self) -> None:
|
||||
"""Wait for the broadcasts to land before returning, so a caller may
|
||||
rebuild or free the patch tensors as soon as `send_weights` returns
|
||||
rather than relying on same-stream ordering. See
|
||||
rebuild or free the patch tensors as soon as a send method returns rather
|
||||
than relying on same-stream ordering. See
|
||||
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
|
||||
cross-rank barrier (and sparse is single-rank on the trainer anyway)."""
|
||||
cross-rank barrier."""
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user