Weight sync refactor + move sparse nccl engine (#44353)

Signed-off-by: hao-aaron <ahao@anyscale.com>
Signed-off-by: haoaaron <ahao@anyscale.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Aaron Hao
2026-07-01 01:25:19 -07:00
committed by GitHub
co-authored by Claude Sonnet 4.6
parent f651a8a9a4
commit 77a9c5ae28
32 changed files with 951 additions and 1044 deletions
+60 -74
View File
@@ -3,12 +3,15 @@
"""Base class for weight transfer engines."""
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterator
from dataclasses import KW_ONLY, dataclass, field
from typing import Any, Generic, Literal, TypeVar
from collections.abc import Iterator
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Generic, TypeVar
import torch
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
@@ -28,44 +31,7 @@ class WeightTransferInitInfo(ABC): # noqa: B024
class WeightTransferUpdateInfo(ABC): # noqa: B024
"""Base class for backend-specific weight update info."""
_: KW_ONLY
update_kind: Literal["dense", "sparse_flat"] = "dense"
"""Weight update format."""
num_updates_list: list[int] | None = None
"""Number of sparse entries to receive for each parameter in ``names``."""
def __post_init__(self) -> None:
if self.update_kind not in ("dense", "sparse_flat"):
raise ValueError(f"Unsupported update_kind: {self.update_kind}")
if self.update_kind == "dense":
if self.num_updates_list is not None:
raise ValueError(
"Sparse metadata is only supported for `update_kind='sparse_flat'`"
)
return
if self.num_updates_list is None:
raise ValueError("`num_updates_list` is required for sparse updates")
if len(self.num_updates_list) == 0:
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
if any(num_updates < 0 for num_updates in self.num_updates_list):
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
names = getattr(self, "names", None)
if names is not None and len(self.num_updates_list) != len(names):
raise ValueError(
f"`num_updates_list` should be of the same size as `names`: "
f"got {len(self.num_updates_list)} and {len(names)}"
)
@dataclass
class SparseWeightPatch:
"""A sparse in-place patch for one existing parameter."""
name: str
indices: torch.Tensor
values: torch.Tensor
pass
# API-level request classes (accept dicts for backend-agnostic serialization)
@@ -89,9 +55,15 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
from a trainer to inference workers.
This abstraction separates weight transfer transport logic from the worker
implementation, allowing different backends (NCCL, CUDA IPC[TODO], RDMA[TODO]) to be
implementation, allowing different backends (NCCL, CUDA IPC, RDMA[TODO]) to be
plugged in.
Each engine owns its full weight-update lifecycle: `start_weight_update`,
`update_weights`, and `finish_weight_update`. Layerwise reloading (used by
checkpoint-format engines) is opted into per engine by running it inside
`start_weight_update`/`finish_weight_update`. Engines that apply weights in
place (e.g. sparse patches) leave those methods as no-ops.
Subclasses should define:
init_info_cls: Type of backend-specific initialization info
update_info_cls: Type of backend-specific update info
@@ -104,7 +76,8 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
def __init__(
self,
config: WeightTransferConfig,
parallel_config: ParallelConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
"""
@@ -112,11 +85,15 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
Args:
config: The configuration for the weight transfer engine
parallel_config: The configuration for the parallel setup
vllm_config: The full vLLM config (provides parallel/model config)
device: The device this worker's model lives on
model: The local model instance which will receive the weights
"""
self.config = config
self.parallel_config = parallel_config
self.vllm_config = vllm_config
self.parallel_config: ParallelConfig = vllm_config.parallel_config
self.model_config = vllm_config.model_config
self.device = device
self.model = model
def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo:
@@ -171,32 +148,50 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
raise NotImplementedError
@abstractmethod
def receive_weights(
self,
update_info: TUpdateInfo,
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
) -> None:
def start_weight_update(self) -> None:
"""
Receive weights from the trainer and load them incrementally.
Prepare the engine for a new weight update.
Engines that receive weights in checkpoint format initialize layerwise reloading
here, else this is typically a no-op.
See: https://docs.vllm.ai/en/latest/training/layerwise/ for more details.
"""
raise NotImplementedError
@abstractmethod
def finish_weight_update(self) -> None:
"""
Finalize the current weight update.
Checkpoint-format engines finalize layerwise reloading here; engines
that apply weights in place leave this as a no-op.
"""
raise NotImplementedError
def update_weights(self, update_info: dict[str, Any]) -> None:
"""
Receive one weight update chunk and load it into the model.
Args:
update_info: Dictionary containing backend-specific update info
"""
typed_update_info = self.parse_update_info(update_info)
self.receive_weights(typed_update_info)
# NCCL broadcast / IPC paths may be asynchronous. Synchronize here so the
# next step uses the new weights.
torch.accelerator.synchronize()
@abstractmethod
def receive_weights(self, update_info: TUpdateInfo) -> None:
"""
Receive weights from the trainer and load them into the model.
Args:
update_info: Backend-specific update info containing parameter metadata
and any backend-specific data
load_weights: Callable that loads weights into the model. Called
incrementally for each weight to avoid OOM.
"""
raise NotImplementedError
def receive_sparse_weights(
self,
update_info: TUpdateInfo,
apply_patches: Callable[[list[SparseWeightPatch]], None],
) -> None:
"""Receive sparse weight patches from the trainer."""
raise NotImplementedError(
f"{self.__class__.__name__} does not support sparse weight updates"
)
@abstractmethod
def shutdown(self) -> None:
"""
@@ -208,7 +203,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
@staticmethod
@abstractmethod
def trainer_send_weights(
iterator: Iterator[tuple[str, torch.Tensor]],
iterator: Iterator[Any],
trainer_args: dict[str, Any] | Any,
) -> None:
"""
@@ -218,8 +213,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
to send weights to all inference workers.
Args:
iterator: Iterator of model parameters. Returns (name, tensor) tuples.
The tensors should be on the appropriate device for the backend.
iterator: Iterator of backend-specific items to send.
trainer_args: Dictionary containing backend-specific arguments needed
to send weights. The structure depends on the backend:
- NCCL: Contains 'group', 'src', 'packed', etc.
@@ -231,11 +225,3 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
>>> engine.trainer_send_weights(param_iter, trainer_args)
"""
raise NotImplementedError
@staticmethod
def trainer_send_sparse_weights(
_iterator: Iterator[SparseWeightPatch],
_trainer_args: dict[str, Any] | Any,
) -> None:
"""Send sparse weight patches from trainer to inference workers."""
raise NotImplementedError("Sparse weight updates are not supported")
+12 -4
View File
@@ -12,7 +12,7 @@ from vllm.logger import init_logger
if TYPE_CHECKING:
import torch
from vllm.config.parallel import ParallelConfig
from vllm.config import VllmConfig
from vllm.config.weight_transfer import WeightTransferConfig
logger = init_logger(__name__)
@@ -76,14 +76,16 @@ class WeightTransferEngineFactory:
def create_engine(
cls,
config: "WeightTransferConfig",
parallel_config: "ParallelConfig",
vllm_config: "VllmConfig",
device: "torch.device",
model: "torch.nn.Module",
) -> WeightTransferEngine:
"""Create a weight transfer engine instance.
Args:
config: Weight transfer configuration containing the backend name
parallel_config: Parallel configuration for the engine
vllm_config: The full vLLM config (provides parallel/model config)
device: The device this worker's model lives on
model: The local model instance which will receive the weights
Returns:
@@ -106,7 +108,7 @@ class WeightTransferEngineFactory:
engine_cls.__name__,
)
return engine_cls(config, parallel_config, model)
return engine_cls(config, vllm_config, device, model)
# Register built-in weight transfer engines here.
@@ -124,3 +126,9 @@ WeightTransferEngineFactory.register_engine(
"vllm.distributed.weight_transfer.ipc_engine",
"IPCWeightTransferEngine",
)
WeightTransferEngineFactory.register_engine(
"sparse_nccl",
"vllm.distributed.weight_transfer.sparse_nccl_engine",
"SparseNCCLWeightTransferEngine",
)
+34 -20
View File
@@ -5,7 +5,7 @@
import pickle
from collections.abc import Callable, Iterator
from dataclasses import asdict, dataclass
from typing import Any
from typing import TYPE_CHECKING, Any
import pybase64 as base64
import ray
@@ -14,13 +14,15 @@ import torch
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
from vllm import envs
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferEngine,
WeightTransferInitInfo,
WeightTransferUpdateInfo,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
packed_ipc_consumer,
@@ -87,10 +89,6 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Whether this update uses packed tensor format."""
def __post_init__(self):
super().__post_init__()
if self.update_kind != "dense":
raise NotImplementedError("IPC weight transfer only supports dense updates")
if self.ipc_handles_pickled is not None:
if self.ipc_handles is not None:
raise ValueError(
@@ -152,7 +150,8 @@ class IPCWeightTransferEngine(
def __init__(
self,
config: WeightTransferConfig,
parallel_config: ParallelConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
"""
@@ -160,10 +159,11 @@ class IPCWeightTransferEngine(
Args:
config: The configuration for the weight transfer engine
parallel_config: The configuration for the parallel setup
vllm_config: The full vLLM config
device: The device this worker's model lives on
model: The local model instance which will receive the weights
"""
super().__init__(config, parallel_config, model)
super().__init__(config, vllm_config, device, model)
def parse_update_info(
self, update_dict: dict[str, Any]
@@ -206,22 +206,37 @@ class IPCWeightTransferEngine(
"""
pass
def receive_weights(
self,
update_info: IPCWeightTransferUpdateInfo,
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
) -> None:
def start_weight_update(self) -> None:
"""Initialize layerwise reloading for the incoming checkpoint weights."""
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
initialize_layerwise_reload(self.model)
def finish_weight_update(self) -> None:
"""Finalize layerwise reloading after all weights have been received."""
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
)
finalize_layerwise_reload(self.model, self.model_config)
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
"""
Receive weights from the trainer via CUDA IPC handles.
Receive weights from the trainer via CUDA IPC handles and load them.
Args:
update_info: IPC update info containing parameter names, dtypes, shapes,
and IPC handles. Each IPC handle is a mapping between physical
GPU UUID and the rebuild_cuda_tensor args tuple.
load_weights: Callable that loads weights into the model. Called
incrementally for each weight to avoid OOM.
"""
device_index = torch.accelerator.current_device_index()
# Use the worker's assigned device rather than the ambient current
# device: the receive path is no longer wrapped in
# `with torch.device(self.device)` by the caller, so the current device
# is not guaranteed to match self.device. The IPC tensors must be
# rebuilt on the device the model lives on.
device_index = self.device.index
if update_info.packed:
assert update_info.tensor_sizes is not None
@@ -234,7 +249,6 @@ class IPCWeightTransferEngine(
tensor_sizes=update_info.tensor_sizes,
device_index=device_index,
)
load_weights(weights)
else:
assert isinstance(update_info.ipc_handles, list)
weights = []
@@ -260,7 +274,7 @@ class IPCWeightTransferEngine(
weight = rebuild_cuda_tensor(*list_args)
weights.append((name, weight))
load_weights(weights)
self.model.load_weights(weights)
def shutdown(self) -> None:
pass
@@ -0,0 +1,120 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared NCCL initialization helpers for weight transfer engines.
The dense (`NCCLWeightTransferEngine`) and sparse
(`SparseNCCLWeightTransferEngine`) backends are independent engines that share
*only* their process-group initialization. That common logic lives here so the
sparse engine does not have to subclass the dense one.
"""
from dataclasses import dataclass
from typing import TYPE_CHECKING
import torch
if TYPE_CHECKING:
from vllm.config.parallel import ParallelConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
@dataclass
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
"""Initialization info for NCCL-based weight transfer backends."""
master_address: str
master_port: int
rank_offset: int
world_size: int
def stateless_init_process_group(
master_address: str,
master_port: int,
rank: int,
world_size: int,
device,
) -> "PyNcclCommunicator":
"""
vLLM provides `StatelessProcessGroup` to create a process group
without considering the global process group in torch.distributed.
It is recommended to create `StatelessProcessGroup`, and then initialize
the data-plane communication (NCCL) between external (train processes)
and vLLM workers.
"""
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
pg = StatelessProcessGroup.create(
host=master_address, port=master_port, rank=rank, world_size=world_size
)
return PyNcclCommunicator(pg, device=device)
def worker_init_process_group(
init_info: NCCLWeightTransferInitInfo,
parallel_config: "ParallelConfig",
) -> "PyNcclCommunicator":
"""Create the trainer<->worker NCCL group on an inference worker.
Computes a unique rank for this worker across all data-parallel groups and
joins the stateless process group with the trainer.
"""
# Calculate the global rank in the trainer-worker process group.
# Must account for data parallel to get unique ranks across all workers.
dp_rank = parallel_config.data_parallel_index
world_size_per_dp = parallel_config.world_size # TP * PP
rank_within_dp = parallel_config.rank
# Unique rank across all DP groups
worker_rank = dp_rank * world_size_per_dp + rank_within_dp
rank = worker_rank + init_info.rank_offset
device = torch.accelerator.current_device_index()
return stateless_init_process_group(
init_info.master_address,
init_info.master_port,
rank,
init_info.world_size,
device=device,
)
def trainer_init(
init_info: NCCLWeightTransferInitInfo | dict,
) -> "PyNcclCommunicator":
"""
Initialize NCCL process group for trainer-side weight transfer.
The trainer is always rank 0 in the process group. Uses the current
CUDA device (torch.accelerator.current_device_index()).
Args:
init_info: Either an NCCLWeightTransferInitInfo object or a dict with keys:
- master_address: str
- master_port: int
- world_size: int
Returns:
PyNcclCommunicator for weight transfer.
"""
if isinstance(init_info, dict):
master_address = init_info["master_address"]
master_port = init_info["master_port"]
world_size = init_info["world_size"]
else:
master_address = init_info.master_address
master_port = init_info.master_port
world_size = init_info.world_size
# Trainer is always rank 0
device = torch.accelerator.current_device_index()
return stateless_init_process_group(
master_address,
master_port,
0,
world_size,
device,
)
+49 -197
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NCCL-based weight transfer engine."""
"""NCCL-based (dense) weight transfer engine."""
from collections.abc import Callable, Iterator
from dataclasses import dataclass
@@ -9,31 +9,32 @@ from typing import TYPE_CHECKING, Any
import torch
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
SparseWeightPatch,
WeightTransferEngine,
WeightTransferInitInfo,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
trainer_init,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
DEFAULT_PACKED_NUM_BUFFERS,
packed_nccl_broadcast_consumer,
)
@dataclass
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
"""Initialization info for NCCL weight transfer backend."""
master_address: str
master_port: int
rank_offset: int
world_size: int
# Re-exported for backward compatibility; canonical home is nccl_common.
__all__ = [
"NCCLWeightTransferInitInfo",
"NCCLTrainerSendWeightsArgs",
"NCCLWeightTransferUpdateInfo",
"NCCLWeightTransferEngine",
]
@dataclass
@@ -82,7 +83,6 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
def __post_init__(self):
"""Validate that all lists have the same length."""
super().__post_init__()
num_params = len(self.names)
if len(self.dtype_names) != num_params:
raise ValueError(
@@ -94,13 +94,6 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
f"`shapes` should be of the same size as `names`: "
f"got {len(self.shapes)} and {len(self.names)}"
)
if self.update_kind == "dense":
return
if self.packed:
raise ValueError(
"`update_kind='sparse_flat'` cannot be combined with `packed=True`"
)
class NCCLWeightTransferEngine(
@@ -109,8 +102,10 @@ class NCCLWeightTransferEngine(
"""
Weight transfer engine using NCCL for communication between trainer and workers.
This implementation uses NCCL broadcast operations to transfer weights from
the trainer (rank 0) to all inference workers in a process group.
This implementation uses NCCL broadcast operations to transfer dense
checkpoint-format weights from the trainer (rank 0) to all inference workers
in a process group. Received weights are loaded via the model's
`load_weights` using the layerwise reload lifecycle.
"""
# Define backend-specific dataclass types
@@ -120,18 +115,11 @@ class NCCLWeightTransferEngine(
def __init__(
self,
config: WeightTransferConfig,
parallel_config: ParallelConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
"""
Initialize the NCCL weight transfer engine.
Args:
config: The configuration for the weight transfer engine
parallel_config: The configuration for the parallel setup
model: The local model instance which will receive the weights
"""
super().__init__(config, parallel_config, model)
super().__init__(config, vllm_config, device, model)
self.model_update_group: PyNcclCommunicator | None = None
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
@@ -142,35 +130,29 @@ class NCCLWeightTransferEngine(
init_info: NCCL initialization info containing master address, port,
rank offset, and world size
"""
# Calculate the global rank in the trainer-worker process group
# Must account for data parallel to get unique ranks across all workers
dp_rank = self.parallel_config.data_parallel_index
world_size_per_dp = self.parallel_config.world_size # TP * PP
rank_within_dp = self.parallel_config.rank
# Unique rank across all DP groups
worker_rank = dp_rank * world_size_per_dp + rank_within_dp
rank = worker_rank + init_info.rank_offset
# Create stateless process group
device = torch.accelerator.current_device_index()
self.model_update_group = (
NCCLWeightTransferEngine._stateless_init_process_group(
init_info.master_address,
init_info.master_port,
rank,
init_info.world_size,
device=device,
)
self.model_update_group = worker_init_process_group(
init_info, self.parallel_config
)
def receive_weights(
self,
update_info: NCCLWeightTransferUpdateInfo,
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
) -> None:
def start_weight_update(self) -> None:
"""Initialize layerwise reloading for the incoming checkpoint weights."""
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
initialize_layerwise_reload(self.model)
def finish_weight_update(self) -> None:
"""Finalize layerwise reloading after all weights have been received."""
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
)
finalize_layerwise_reload(self.model, self.model_config)
def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None:
"""
Receive weights from trainer via NCCL broadcast and load them incrementally.
Receive weights from trainer via NCCL broadcast.
If update_info.packed is True, uses packed tensor broadcasting for
efficient transfer of multiple weights in batches. Otherwise, uses simple
@@ -179,19 +161,12 @@ class NCCLWeightTransferEngine(
Args:
update_info: NCCL update info containing parameter names, dtypes, shapes,
and packed flag
load_weights: Callable that loads weights into the model. Called
incrementally for each batch of weights to avoid OOM.
"""
if self.model_update_group is None:
raise RuntimeError(
"NCCL weight transfer not initialized. "
"Call init_transfer_engine() first."
)
if update_info.update_kind != "dense":
raise ValueError(
"Sparse updates must use `receive_sparse_weights`, not "
"`receive_weights`"
)
if update_info.packed:
# Build iterator of (name, (shape, dtype)) from update_info
@@ -206,9 +181,10 @@ class NCCLWeightTransferEngine(
iterator=state_dict_info_iterator(),
group=self.model_update_group,
src=0,
post_unpack_func=load_weights,
post_unpack_func=self.model.load_weights,
buffer_size_bytes=update_info.packed_buffer_size_bytes,
num_buffers=update_info.packed_num_buffers,
device=self.device,
)
else:
# Use simple one-by-one broadcasting
@@ -216,49 +192,13 @@ class NCCLWeightTransferEngine(
update_info.names, update_info.dtype_names, update_info.shapes
):
dtype = getattr(torch, dtype_name)
weight = torch.empty(shape, dtype=dtype, device="cuda")
weight = torch.empty(shape, dtype=dtype, device=self.device)
self.model_update_group.broadcast(
weight, src=0, stream=torch.cuda.current_stream()
)
load_weights([(name, weight)])
self.model.load_weights([(name, weight)])
del weight
def receive_sparse_weights(
self,
update_info: NCCLWeightTransferUpdateInfo,
apply_patches: Callable[[list[SparseWeightPatch]], None],
) -> None:
"""Receive sparse flat-index patches from trainer via NCCL."""
if self.model_update_group is None:
raise RuntimeError(
"NCCL weight transfer not initialized. "
"Call init_transfer_engine() first."
)
if update_info.update_kind != "sparse_flat":
raise ValueError("Sparse receive path requires `update_kind='sparse_flat'`")
assert update_info.num_updates_list is not None
for name, dtype_name, num_updates in zip(
update_info.names,
update_info.dtype_names,
update_info.num_updates_list,
):
dtype = getattr(torch, dtype_name)
device = torch.accelerator.current_device_index()
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
values = torch.empty(num_updates, dtype=dtype, device=device)
self.model_update_group.broadcast(
indices, src=0, stream=torch.cuda.current_stream()
)
self.model_update_group.broadcast(
values, src=0, stream=torch.cuda.current_stream()
)
apply_patches(
[SparseWeightPatch(name=name, indices=indices, values=values)]
)
del indices
del values
def shutdown(self) -> None:
if self.model_update_group is not None:
# Clean up the communicator by removing the reference
@@ -269,7 +209,7 @@ class NCCLWeightTransferEngine(
iterator: Iterator[tuple[str, torch.Tensor]],
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
) -> None:
"""Broadcast weights from trainer to vLLM workers.
"""Broadcast dense weights from trainer to vLLM workers.
Args:
iterator: Iterator of model parameters. Returns (name, tensor) tuples
@@ -322,94 +262,6 @@ class NCCLWeightTransferEngine(
stream=args.stream or torch.cuda.current_stream(),
)
@staticmethod
def trainer_send_sparse_weights(
iterator: Iterator[SparseWeightPatch],
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
) -> None:
"""Broadcast sparse flat-index patches from trainer to vLLM workers."""
if isinstance(trainer_args, dict):
args = NCCLTrainerSendWeightsArgs(**trainer_args)
else:
args = trainer_args
if args.packed:
raise ValueError(
"Sparse NCCL updates cannot be combined with `packed=True`"
)
stream = args.stream or torch.cuda.current_stream()
for patch in iterator:
args.group.broadcast(patch.indices, src=args.src, stream=stream)
args.group.broadcast(patch.values, src=args.src, stream=stream)
@staticmethod
def trainer_init(
init_info: NCCLWeightTransferInitInfo | dict,
) -> "PyNcclCommunicator":
"""
Initialize NCCL process group for trainer-side weight transfer.
The trainer is always rank 0 in the process group. Uses the current
CUDA device (torch.accelerator.current_device_index()).
Args:
init_info: Either an NCCLWeightTransferInitInfo object or a dict with keys:
- master_address: str
- master_port: int
- world_size: int
Returns:
PyNcclCommunicator for weight transfer.
Example:
>>> from vllm.distributed.weight_transfer.nccl_engine import (
... NCCLWeightTransferEngine,
... )
>>> group = NCCLWeightTransferEngine.trainer_init(
... dict(
... master_address=master_address,
... master_port=master_port,
... world_size=world_size,
... ),
... )
"""
if isinstance(init_info, dict):
master_address = init_info["master_address"]
master_port = init_info["master_port"]
world_size = init_info["world_size"]
else:
# NCCLWeightTransferInitInfo object
master_address = init_info.master_address
master_port = init_info.master_port
world_size = init_info.world_size
# Trainer is always rank 0
device = torch.accelerator.current_device_index()
return NCCLWeightTransferEngine._stateless_init_process_group(
master_address,
master_port,
0,
world_size,
device,
)
@staticmethod
def _stateless_init_process_group(
master_address, master_port, rank, world_size, device
):
"""
vLLM provides `StatelessProcessGroup` to create a process group
without considering the global process group in torch.distributed.
It is recommended to create `StatelessProcessGroup`, and then initialize
the data-plane communication (NCCL) between external (train processes)
and vLLM workers.
"""
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
pg = StatelessProcessGroup.create(
host=master_address, port=master_port, rank=rank, world_size=world_size
)
pynccl = PyNcclCommunicator(pg, device=device)
return pynccl
# Trainer-side process-group setup. Delegates to the shared helper so the
# sparse engine can reuse the exact same rendezvous without subclassing.
trainer_init = staticmethod(trainer_init)
@@ -186,6 +186,7 @@ def packed_nccl_broadcast_consumer(
post_unpack_func: Callable[[list[tuple[str, torch.Tensor]]], None],
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
device: torch.device | str = "cuda",
) -> None:
"""Consume packed tensors and unpack them into a list of tensors.
@@ -199,6 +200,8 @@ def packed_nccl_broadcast_consumer(
Both producer and consumer must use the same value.
num_buffers: Number of buffers for double/triple buffering.
Both producer and consumer must use the same value.
device: Device for the receive buffers. Must match the device the NCCL
communicator was created on (the worker's assigned device).
"""
target_packed_tensor_size = buffer_size_bytes
@@ -211,7 +214,7 @@ def packed_nccl_broadcast_consumer(
]
packing_tensor_sizes: list[int] = [0 for _ in range(num_buffers)]
packed_tensors: list[torch.Tensor] = [
torch.empty(0, dtype=torch.uint8, device="cuda") for _ in range(num_buffers)
torch.empty(0, dtype=torch.uint8, device=device) for _ in range(num_buffers)
]
while True:
@@ -234,7 +237,7 @@ def packed_nccl_broadcast_consumer(
break
# Create a packed tensor and broadcast it
packed_tensors[buffer_idx] = torch.empty(
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device="cuda"
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device=device
)
group.broadcast(packed_tensors[buffer_idx], src=src)
# Load the packed tensor into the model
@@ -259,7 +262,7 @@ def packed_nccl_broadcast_consumer(
packed_tensors[buffer_idx] = torch.empty(
packing_tensor_sizes[buffer_idx],
dtype=torch.uint8,
device="cuda",
device=device,
)
group.broadcast(packed_tensors[buffer_idx], src=src)
# Load the packed tensor into the model
@@ -0,0 +1,223 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sparse NCCL weight transfer engine.
A standalone engine (not a subclass of `NCCLWeightTransferEngine`) for applying
sparse, flat-index weight patches in place. It shares only NCCL process-group
initialization with the dense engine (via `nccl_common`); the update path
applies index/value patches directly to existing model parameters and never runs
layerwise reload.
MVP limitations:
* TP=1 and PP=1 only
* uses runtime/kernel-format parameter names
* not composable with checkpoint-format or packed updates
"""
from collections.abc import Iterator
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import torch
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferEngine,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
trainer_init,
worker_init_process_group,
)
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerSendWeightsArgs
__all__ = [
"SparseWeightPatch",
"SparseNCCLWeightTransferUpdateInfo",
"SparseNCCLWeightTransferEngine",
]
@dataclass
class SparseWeightPatch:
"""A sparse in-place patch for one existing parameter."""
name: str
indices: torch.Tensor
values: torch.Tensor
@dataclass
class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Update info for the sparse NCCL weight transfer backend."""
names: list[str]
dtype_names: list[str]
shapes: list[list[int]]
num_updates_list: list[int]
"""Number of sparse entries to receive for each parameter in ``names``."""
def __post_init__(self) -> None:
num_params = len(self.names)
if len(self.dtype_names) != num_params:
raise ValueError(
f"`dtype_names` should be of the same size as `names`: "
f"got {len(self.dtype_names)} and {len(self.names)}"
)
if len(self.shapes) != num_params:
raise ValueError(
f"`shapes` should be of the same size as `names`: "
f"got {len(self.shapes)} and {len(self.names)}"
)
if len(self.num_updates_list) == 0:
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
if len(self.num_updates_list) != num_params:
raise ValueError(
f"`num_updates_list` should be of the same size as `names`: "
f"got {len(self.num_updates_list)} and {len(self.names)}"
)
if any(num_updates < 0 for num_updates in self.num_updates_list):
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
class SparseNCCLWeightTransferEngine(
WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo]
):
"""
Sparse weight transfer engine using NCCL.
Receives flat-index (indices, values) patches broadcast from the trainer
(rank 0) and applies them in place to existing model parameters. Weights are
applied directly without layerwise reload, so `start_weight_update` and
`finish_weight_update` are no-ops.
"""
init_info_cls = NCCLWeightTransferInitInfo
update_info_cls = SparseNCCLWeightTransferUpdateInfo
def __init__(
self,
config: WeightTransferConfig,
vllm_config: "VllmConfig",
device: torch.device,
model: torch.nn.Module,
) -> None:
super().__init__(config, vllm_config, device, model)
self.model_update_group: PyNcclCommunicator | None = None
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
"""Initialize the NCCL process group with the trainer."""
self.model_update_group = worker_init_process_group(
init_info, self.parallel_config
)
def start_weight_update(self) -> None:
"""No-op: sparse patches are applied in place, no layerwise reload."""
if self.parallel_config.world_size != 1:
raise NotImplementedError(
"Sparse weight updates currently require TP=1 and PP=1"
)
def finish_weight_update(self) -> None:
"""No-op: sparse patches are applied in place, no layerwise reload."""
pass
def receive_weights(self, update_info: SparseNCCLWeightTransferUpdateInfo) -> None:
"""Receive sparse flat-index patches from the trainer and apply them."""
if self.model_update_group is None:
raise RuntimeError(
"NCCL weight transfer not initialized. "
"Call init_transfer_engine() first."
)
# Use the worker's assigned device rather than the ambient current
# device: the receive path is no longer wrapped in
# `with torch.device(self.device)` by the caller, so the current device
# is not guaranteed to match self.device. The recv buffers must live on
# the same device as the NCCL communicator (created on self.device).
device = self.device
for name, dtype_name, num_updates in zip(
update_info.names,
update_info.dtype_names,
update_info.num_updates_list,
):
dtype = getattr(torch, dtype_name)
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
values = torch.empty(num_updates, dtype=dtype, device=device)
self.model_update_group.broadcast(
indices, src=0, stream=torch.cuda.current_stream()
)
self.model_update_group.broadcast(
values, src=0, stream=torch.cuda.current_stream()
)
self._apply_patch(
SparseWeightPatch(name=name, indices=indices, values=values)
)
del indices
del values
def _apply_patch(self, patch: SparseWeightPatch) -> None:
"""Apply a single sparse flat-index patch to an existing model param."""
param = self.model.get_parameter(patch.name)
if not param.data.is_contiguous():
raise NotImplementedError(
"Sparse weight updates currently require contiguous params: "
f"{patch.name}"
)
if patch.indices.dtype != torch.int32:
raise ValueError(
f"Sparse weight updates currently require int32 indices: {patch.name}"
)
if patch.indices.ndim != 1 or patch.values.ndim != 1:
raise ValueError(
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
)
if patch.indices.numel() != patch.values.numel():
raise ValueError(
f"`indices` and `values` must have matching lengths for {patch.name}"
)
if patch.values.dtype != param.dtype:
raise ValueError(
f"Sparse values dtype {patch.values.dtype} does not match "
f"parameter dtype {param.dtype} for {patch.name}"
)
flat_param = param.data.view(-1)
flat_param.index_copy_(
0,
patch.indices.to(device=flat_param.device, dtype=torch.long),
patch.values.to(device=flat_param.device),
)
def shutdown(self) -> None:
if self.model_update_group is not None:
self.model_update_group = None
@staticmethod
def trainer_send_weights(
iterator: Iterator[SparseWeightPatch],
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
) -> None:
"""Broadcast sparse flat-index patches from trainer to vLLM workers."""
if isinstance(trainer_args, dict):
args = NCCLTrainerSendWeightsArgs(**trainer_args)
else:
args = trainer_args
if args.packed:
raise ValueError(
"Sparse NCCL updates cannot be combined with `packed=True`"
)
stream = args.stream or torch.cuda.current_stream()
for patch in iterator:
args.group.broadcast(patch.indices, src=args.src, stream=stream)
args.group.broadcast(patch.values, src=args.src, stream=stream)
# Trainer-side process-group setup (shared with the dense engine).
trainer_init = staticmethod(trainer_init)