Weight sync refactor + move sparse nccl engine (#44353)

Signed-off-by: hao-aaron <ahao@anyscale.com>
Signed-off-by: haoaaron <ahao@anyscale.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Aaron Hao
2026-07-01 01:25:19 -07:00
committed by GitHub
co-authored by Claude Sonnet 4.6
parent f651a8a9a4
commit 77a9c5ae28
32 changed files with 951 additions and 1044 deletions
+34 -20
View File
@@ -5,7 +5,7 @@
import pickle
from collections.abc import Callable, Iterator
from dataclasses import asdict, dataclass
from typing import Any
from typing import TYPE_CHECKING, Any
import pybase64 as base64
import ray
@@ -14,13 +14,15 @@ import torch
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
from vllm import envs
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferEngine,
WeightTransferInitInfo,
WeightTransferUpdateInfo,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
packed_ipc_consumer,
@@ -87,10 +89,6 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Whether this update uses packed tensor format."""
def __post_init__(self):
super().__post_init__()
if self.update_kind != "dense":
raise NotImplementedError("IPC weight transfer only supports dense updates")
if self.ipc_handles_pickled is not None:
if self.ipc_handles is not None:
raise ValueError(
@@ -152,7 +150,8 @@ class IPCWeightTransferEngine(
def __init__(
self,
config: WeightTransferConfig,
parallel_config: ParallelConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
"""
@@ -160,10 +159,11 @@ class IPCWeightTransferEngine(
Args:
config: The configuration for the weight transfer engine
parallel_config: The configuration for the parallel setup
vllm_config: The full vLLM config
device: The device this worker's model lives on
model: The local model instance which will receive the weights
"""
super().__init__(config, parallel_config, model)
super().__init__(config, vllm_config, device, model)
def parse_update_info(
self, update_dict: dict[str, Any]
@@ -206,22 +206,37 @@ class IPCWeightTransferEngine(
"""
pass
def receive_weights(
self,
update_info: IPCWeightTransferUpdateInfo,
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
) -> None:
def start_weight_update(self) -> None:
"""Initialize layerwise reloading for the incoming checkpoint weights."""
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
initialize_layerwise_reload(self.model)
def finish_weight_update(self) -> None:
"""Finalize layerwise reloading after all weights have been received."""
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
)
finalize_layerwise_reload(self.model, self.model_config)
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
"""
Receive weights from the trainer via CUDA IPC handles.
Receive weights from the trainer via CUDA IPC handles and load them.
Args:
update_info: IPC update info containing parameter names, dtypes, shapes,
and IPC handles. Each IPC handle is a mapping between physical
GPU UUID and the rebuild_cuda_tensor args tuple.
load_weights: Callable that loads weights into the model. Called
incrementally for each weight to avoid OOM.
"""
device_index = torch.accelerator.current_device_index()
# Use the worker's assigned device rather than the ambient current
# device: the receive path is no longer wrapped in
# `with torch.device(self.device)` by the caller, so the current device
# is not guaranteed to match self.device. The IPC tensors must be
# rebuilt on the device the model lives on.
device_index = self.device.index
if update_info.packed:
assert update_info.tensor_sizes is not None
@@ -234,7 +249,6 @@ class IPCWeightTransferEngine(
tensor_sizes=update_info.tensor_sizes,
device_index=device_index,
)
load_weights(weights)
else:
assert isinstance(update_info.ipc_handles, list)
weights = []
@@ -260,7 +274,7 @@ class IPCWeightTransferEngine(
weight = rebuild_cuda_tensor(*list_args)
weights.append((name, weight))
load_weights(weights)
self.model.load_weights(weights)
def shutdown(self) -> None:
pass