Weight sync refactor + move sparse nccl engine (#44353)
Signed-off-by: hao-aaron <ahao@anyscale.com> Signed-off-by: haoaaron <ahao@anyscale.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
f651a8a9a4
commit
77a9c5ae28
@@ -3,12 +3,15 @@
|
||||
"""Base class for weight transfer engines."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import KW_ONLY, dataclass, field
|
||||
from typing import Any, Generic, Literal, TypeVar
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Generic, TypeVar
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
|
||||
@@ -28,44 +31,7 @@ class WeightTransferInitInfo(ABC): # noqa: B024
|
||||
class WeightTransferUpdateInfo(ABC): # noqa: B024
|
||||
"""Base class for backend-specific weight update info."""
|
||||
|
||||
_: KW_ONLY
|
||||
update_kind: Literal["dense", "sparse_flat"] = "dense"
|
||||
"""Weight update format."""
|
||||
num_updates_list: list[int] | None = None
|
||||
"""Number of sparse entries to receive for each parameter in ``names``."""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.update_kind not in ("dense", "sparse_flat"):
|
||||
raise ValueError(f"Unsupported update_kind: {self.update_kind}")
|
||||
if self.update_kind == "dense":
|
||||
if self.num_updates_list is not None:
|
||||
raise ValueError(
|
||||
"Sparse metadata is only supported for `update_kind='sparse_flat'`"
|
||||
)
|
||||
return
|
||||
|
||||
if self.num_updates_list is None:
|
||||
raise ValueError("`num_updates_list` is required for sparse updates")
|
||||
if len(self.num_updates_list) == 0:
|
||||
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
|
||||
if any(num_updates < 0 for num_updates in self.num_updates_list):
|
||||
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
|
||||
|
||||
names = getattr(self, "names", None)
|
||||
if names is not None and len(self.num_updates_list) != len(names):
|
||||
raise ValueError(
|
||||
f"`num_updates_list` should be of the same size as `names`: "
|
||||
f"got {len(self.num_updates_list)} and {len(names)}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseWeightPatch:
|
||||
"""A sparse in-place patch for one existing parameter."""
|
||||
|
||||
name: str
|
||||
indices: torch.Tensor
|
||||
values: torch.Tensor
|
||||
pass
|
||||
|
||||
|
||||
# API-level request classes (accept dicts for backend-agnostic serialization)
|
||||
@@ -89,9 +55,15 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
from a trainer to inference workers.
|
||||
|
||||
This abstraction separates weight transfer transport logic from the worker
|
||||
implementation, allowing different backends (NCCL, CUDA IPC[TODO], RDMA[TODO]) to be
|
||||
implementation, allowing different backends (NCCL, CUDA IPC, RDMA[TODO]) to be
|
||||
plugged in.
|
||||
|
||||
Each engine owns its full weight-update lifecycle: `start_weight_update`,
|
||||
`update_weights`, and `finish_weight_update`. Layerwise reloading (used by
|
||||
checkpoint-format engines) is opted into per engine by running it inside
|
||||
`start_weight_update`/`finish_weight_update`. Engines that apply weights in
|
||||
place (e.g. sparse patches) leave those methods as no-ops.
|
||||
|
||||
Subclasses should define:
|
||||
init_info_cls: Type of backend-specific initialization info
|
||||
update_info_cls: Type of backend-specific update info
|
||||
@@ -104,7 +76,8 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
parallel_config: ParallelConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -112,11 +85,15 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
Args:
|
||||
config: The configuration for the weight transfer engine
|
||||
parallel_config: The configuration for the parallel setup
|
||||
vllm_config: The full vLLM config (provides parallel/model config)
|
||||
device: The device this worker's model lives on
|
||||
model: The local model instance which will receive the weights
|
||||
"""
|
||||
self.config = config
|
||||
self.parallel_config = parallel_config
|
||||
self.vllm_config = vllm_config
|
||||
self.parallel_config: ParallelConfig = vllm_config.parallel_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.device = device
|
||||
self.model = model
|
||||
|
||||
def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo:
|
||||
@@ -171,32 +148,50 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def receive_weights(
|
||||
self,
|
||||
update_info: TUpdateInfo,
|
||||
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
|
||||
) -> None:
|
||||
def start_weight_update(self) -> None:
|
||||
"""
|
||||
Receive weights from the trainer and load them incrementally.
|
||||
Prepare the engine for a new weight update.
|
||||
|
||||
Engines that receive weights in checkpoint format initialize layerwise reloading
|
||||
here, else this is typically a no-op.
|
||||
See: https://docs.vllm.ai/en/latest/training/layerwise/ for more details.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def finish_weight_update(self) -> None:
|
||||
"""
|
||||
Finalize the current weight update.
|
||||
|
||||
Checkpoint-format engines finalize layerwise reloading here; engines
|
||||
that apply weights in place leave this as a no-op.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def update_weights(self, update_info: dict[str, Any]) -> None:
|
||||
"""
|
||||
Receive one weight update chunk and load it into the model.
|
||||
|
||||
Args:
|
||||
update_info: Dictionary containing backend-specific update info
|
||||
"""
|
||||
typed_update_info = self.parse_update_info(update_info)
|
||||
self.receive_weights(typed_update_info)
|
||||
# NCCL broadcast / IPC paths may be asynchronous. Synchronize here so the
|
||||
# next step uses the new weights.
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@abstractmethod
|
||||
def receive_weights(self, update_info: TUpdateInfo) -> None:
|
||||
"""
|
||||
Receive weights from the trainer and load them into the model.
|
||||
|
||||
Args:
|
||||
update_info: Backend-specific update info containing parameter metadata
|
||||
and any backend-specific data
|
||||
load_weights: Callable that loads weights into the model. Called
|
||||
incrementally for each weight to avoid OOM.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def receive_sparse_weights(
|
||||
self,
|
||||
update_info: TUpdateInfo,
|
||||
apply_patches: Callable[[list[SparseWeightPatch]], None],
|
||||
) -> None:
|
||||
"""Receive sparse weight patches from the trainer."""
|
||||
raise NotImplementedError(
|
||||
f"{self.__class__.__name__} does not support sparse weight updates"
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def shutdown(self) -> None:
|
||||
"""
|
||||
@@ -208,7 +203,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def trainer_send_weights(
|
||||
iterator: Iterator[tuple[str, torch.Tensor]],
|
||||
iterator: Iterator[Any],
|
||||
trainer_args: dict[str, Any] | Any,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -218,8 +213,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
to send weights to all inference workers.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of model parameters. Returns (name, tensor) tuples.
|
||||
The tensors should be on the appropriate device for the backend.
|
||||
iterator: Iterator of backend-specific items to send.
|
||||
trainer_args: Dictionary containing backend-specific arguments needed
|
||||
to send weights. The structure depends on the backend:
|
||||
- NCCL: Contains 'group', 'src', 'packed', etc.
|
||||
@@ -231,11 +225,3 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
>>> engine.trainer_send_weights(param_iter, trainer_args)
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def trainer_send_sparse_weights(
|
||||
_iterator: Iterator[SparseWeightPatch],
|
||||
_trainer_args: dict[str, Any] | Any,
|
||||
) -> None:
|
||||
"""Send sparse weight patches from trainer to inference workers."""
|
||||
raise NotImplementedError("Sparse weight updates are not supported")
|
||||
|
||||
@@ -12,7 +12,7 @@ from vllm.logger import init_logger
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -76,14 +76,16 @@ class WeightTransferEngineFactory:
|
||||
def create_engine(
|
||||
cls,
|
||||
config: "WeightTransferConfig",
|
||||
parallel_config: "ParallelConfig",
|
||||
vllm_config: "VllmConfig",
|
||||
device: "torch.device",
|
||||
model: "torch.nn.Module",
|
||||
) -> WeightTransferEngine:
|
||||
"""Create a weight transfer engine instance.
|
||||
|
||||
Args:
|
||||
config: Weight transfer configuration containing the backend name
|
||||
parallel_config: Parallel configuration for the engine
|
||||
vllm_config: The full vLLM config (provides parallel/model config)
|
||||
device: The device this worker's model lives on
|
||||
model: The local model instance which will receive the weights
|
||||
|
||||
Returns:
|
||||
@@ -106,7 +108,7 @@ class WeightTransferEngineFactory:
|
||||
engine_cls.__name__,
|
||||
)
|
||||
|
||||
return engine_cls(config, parallel_config, model)
|
||||
return engine_cls(config, vllm_config, device, model)
|
||||
|
||||
|
||||
# Register built-in weight transfer engines here.
|
||||
@@ -124,3 +126,9 @@ WeightTransferEngineFactory.register_engine(
|
||||
"vllm.distributed.weight_transfer.ipc_engine",
|
||||
"IPCWeightTransferEngine",
|
||||
)
|
||||
|
||||
WeightTransferEngineFactory.register_engine(
|
||||
"sparse_nccl",
|
||||
"vllm.distributed.weight_transfer.sparse_nccl_engine",
|
||||
"SparseNCCLWeightTransferEngine",
|
||||
)
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import pickle
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pybase64 as base64
|
||||
import ray
|
||||
@@ -14,13 +14,15 @@ import torch
|
||||
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferEngine,
|
||||
WeightTransferInitInfo,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
packed_ipc_consumer,
|
||||
@@ -87,10 +89,6 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Whether this update uses packed tensor format."""
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
if self.update_kind != "dense":
|
||||
raise NotImplementedError("IPC weight transfer only supports dense updates")
|
||||
|
||||
if self.ipc_handles_pickled is not None:
|
||||
if self.ipc_handles is not None:
|
||||
raise ValueError(
|
||||
@@ -152,7 +150,8 @@ class IPCWeightTransferEngine(
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
parallel_config: ParallelConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -160,10 +159,11 @@ class IPCWeightTransferEngine(
|
||||
|
||||
Args:
|
||||
config: The configuration for the weight transfer engine
|
||||
parallel_config: The configuration for the parallel setup
|
||||
vllm_config: The full vLLM config
|
||||
device: The device this worker's model lives on
|
||||
model: The local model instance which will receive the weights
|
||||
"""
|
||||
super().__init__(config, parallel_config, model)
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
|
||||
def parse_update_info(
|
||||
self, update_dict: dict[str, Any]
|
||||
@@ -206,22 +206,37 @@ class IPCWeightTransferEngine(
|
||||
"""
|
||||
pass
|
||||
|
||||
def receive_weights(
|
||||
self,
|
||||
update_info: IPCWeightTransferUpdateInfo,
|
||||
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
|
||||
) -> None:
|
||||
def start_weight_update(self) -> None:
|
||||
"""Initialize layerwise reloading for the incoming checkpoint weights."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
|
||||
initialize_layerwise_reload(self.model)
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Finalize layerwise reloading after all weights have been received."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
)
|
||||
|
||||
finalize_layerwise_reload(self.model, self.model_config)
|
||||
|
||||
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
|
||||
"""
|
||||
Receive weights from the trainer via CUDA IPC handles.
|
||||
Receive weights from the trainer via CUDA IPC handles and load them.
|
||||
|
||||
Args:
|
||||
update_info: IPC update info containing parameter names, dtypes, shapes,
|
||||
and IPC handles. Each IPC handle is a mapping between physical
|
||||
GPU UUID and the rebuild_cuda_tensor args tuple.
|
||||
load_weights: Callable that loads weights into the model. Called
|
||||
incrementally for each weight to avoid OOM.
|
||||
"""
|
||||
device_index = torch.accelerator.current_device_index()
|
||||
# Use the worker's assigned device rather than the ambient current
|
||||
# device: the receive path is no longer wrapped in
|
||||
# `with torch.device(self.device)` by the caller, so the current device
|
||||
# is not guaranteed to match self.device. The IPC tensors must be
|
||||
# rebuilt on the device the model lives on.
|
||||
device_index = self.device.index
|
||||
|
||||
if update_info.packed:
|
||||
assert update_info.tensor_sizes is not None
|
||||
@@ -234,7 +249,6 @@ class IPCWeightTransferEngine(
|
||||
tensor_sizes=update_info.tensor_sizes,
|
||||
device_index=device_index,
|
||||
)
|
||||
load_weights(weights)
|
||||
else:
|
||||
assert isinstance(update_info.ipc_handles, list)
|
||||
weights = []
|
||||
@@ -260,7 +274,7 @@ class IPCWeightTransferEngine(
|
||||
weight = rebuild_cuda_tensor(*list_args)
|
||||
weights.append((name, weight))
|
||||
|
||||
load_weights(weights)
|
||||
self.model.load_weights(weights)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
pass
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared NCCL initialization helpers for weight transfer engines.
|
||||
|
||||
The dense (`NCCLWeightTransferEngine`) and sparse
|
||||
(`SparseNCCLWeightTransferEngine`) backends are independent engines that share
|
||||
*only* their process-group initialization. That common logic lives here so the
|
||||
sparse engine does not have to subclass the dense one.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
from vllm.distributed.weight_transfer.base import WeightTransferInitInfo
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Initialization info for NCCL-based weight transfer backends."""
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
rank_offset: int
|
||||
world_size: int
|
||||
|
||||
|
||||
def stateless_init_process_group(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
rank: int,
|
||||
world_size: int,
|
||||
device,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""
|
||||
vLLM provides `StatelessProcessGroup` to create a process group
|
||||
without considering the global process group in torch.distributed.
|
||||
It is recommended to create `StatelessProcessGroup`, and then initialize
|
||||
the data-plane communication (NCCL) between external (train processes)
|
||||
and vLLM workers.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
pg = StatelessProcessGroup.create(
|
||||
host=master_address, port=master_port, rank=rank, world_size=world_size
|
||||
)
|
||||
return PyNcclCommunicator(pg, device=device)
|
||||
|
||||
|
||||
def worker_init_process_group(
|
||||
init_info: NCCLWeightTransferInitInfo,
|
||||
parallel_config: "ParallelConfig",
|
||||
) -> "PyNcclCommunicator":
|
||||
"""Create the trainer<->worker NCCL group on an inference worker.
|
||||
|
||||
Computes a unique rank for this worker across all data-parallel groups and
|
||||
joins the stateless process group with the trainer.
|
||||
"""
|
||||
# Calculate the global rank in the trainer-worker process group.
|
||||
# Must account for data parallel to get unique ranks across all workers.
|
||||
dp_rank = parallel_config.data_parallel_index
|
||||
world_size_per_dp = parallel_config.world_size # TP * PP
|
||||
rank_within_dp = parallel_config.rank
|
||||
|
||||
# Unique rank across all DP groups
|
||||
worker_rank = dp_rank * world_size_per_dp + rank_within_dp
|
||||
rank = worker_rank + init_info.rank_offset
|
||||
|
||||
device = torch.accelerator.current_device_index()
|
||||
return stateless_init_process_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
rank,
|
||||
init_info.world_size,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
def trainer_init(
|
||||
init_info: NCCLWeightTransferInitInfo | dict,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""
|
||||
Initialize NCCL process group for trainer-side weight transfer.
|
||||
|
||||
The trainer is always rank 0 in the process group. Uses the current
|
||||
CUDA device (torch.accelerator.current_device_index()).
|
||||
|
||||
Args:
|
||||
init_info: Either an NCCLWeightTransferInitInfo object or a dict with keys:
|
||||
- master_address: str
|
||||
- master_port: int
|
||||
- world_size: int
|
||||
|
||||
Returns:
|
||||
PyNcclCommunicator for weight transfer.
|
||||
"""
|
||||
if isinstance(init_info, dict):
|
||||
master_address = init_info["master_address"]
|
||||
master_port = init_info["master_port"]
|
||||
world_size = init_info["world_size"]
|
||||
else:
|
||||
master_address = init_info.master_address
|
||||
master_port = init_info.master_port
|
||||
world_size = init_info.world_size
|
||||
|
||||
# Trainer is always rank 0
|
||||
device = torch.accelerator.current_device_index()
|
||||
return stateless_init_process_group(
|
||||
master_address,
|
||||
master_port,
|
||||
0,
|
||||
world_size,
|
||||
device,
|
||||
)
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NCCL-based weight transfer engine."""
|
||||
"""NCCL-based (dense) weight transfer engine."""
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass
|
||||
@@ -9,31 +9,32 @@ from typing import TYPE_CHECKING, Any
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
SparseWeightPatch,
|
||||
WeightTransferEngine,
|
||||
WeightTransferInitInfo,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
trainer_init,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
DEFAULT_PACKED_NUM_BUFFERS,
|
||||
packed_nccl_broadcast_consumer,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NCCLWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
"""Initialization info for NCCL weight transfer backend."""
|
||||
|
||||
master_address: str
|
||||
master_port: int
|
||||
rank_offset: int
|
||||
world_size: int
|
||||
# Re-exported for backward compatibility; canonical home is nccl_common.
|
||||
__all__ = [
|
||||
"NCCLWeightTransferInitInfo",
|
||||
"NCCLTrainerSendWeightsArgs",
|
||||
"NCCLWeightTransferUpdateInfo",
|
||||
"NCCLWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -82,7 +83,6 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate that all lists have the same length."""
|
||||
super().__post_init__()
|
||||
num_params = len(self.names)
|
||||
if len(self.dtype_names) != num_params:
|
||||
raise ValueError(
|
||||
@@ -94,13 +94,6 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
f"`shapes` should be of the same size as `names`: "
|
||||
f"got {len(self.shapes)} and {len(self.names)}"
|
||||
)
|
||||
if self.update_kind == "dense":
|
||||
return
|
||||
|
||||
if self.packed:
|
||||
raise ValueError(
|
||||
"`update_kind='sparse_flat'` cannot be combined with `packed=True`"
|
||||
)
|
||||
|
||||
|
||||
class NCCLWeightTransferEngine(
|
||||
@@ -109,8 +102,10 @@ class NCCLWeightTransferEngine(
|
||||
"""
|
||||
Weight transfer engine using NCCL for communication between trainer and workers.
|
||||
|
||||
This implementation uses NCCL broadcast operations to transfer weights from
|
||||
the trainer (rank 0) to all inference workers in a process group.
|
||||
This implementation uses NCCL broadcast operations to transfer dense
|
||||
checkpoint-format weights from the trainer (rank 0) to all inference workers
|
||||
in a process group. Received weights are loaded via the model's
|
||||
`load_weights` using the layerwise reload lifecycle.
|
||||
"""
|
||||
|
||||
# Define backend-specific dataclass types
|
||||
@@ -120,18 +115,11 @@ class NCCLWeightTransferEngine(
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
parallel_config: ParallelConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the NCCL weight transfer engine.
|
||||
|
||||
Args:
|
||||
config: The configuration for the weight transfer engine
|
||||
parallel_config: The configuration for the parallel setup
|
||||
model: The local model instance which will receive the weights
|
||||
"""
|
||||
super().__init__(config, parallel_config, model)
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
|
||||
@@ -142,35 +130,29 @@ class NCCLWeightTransferEngine(
|
||||
init_info: NCCL initialization info containing master address, port,
|
||||
rank offset, and world size
|
||||
"""
|
||||
|
||||
# Calculate the global rank in the trainer-worker process group
|
||||
# Must account for data parallel to get unique ranks across all workers
|
||||
dp_rank = self.parallel_config.data_parallel_index
|
||||
world_size_per_dp = self.parallel_config.world_size # TP * PP
|
||||
rank_within_dp = self.parallel_config.rank
|
||||
|
||||
# Unique rank across all DP groups
|
||||
worker_rank = dp_rank * world_size_per_dp + rank_within_dp
|
||||
rank = worker_rank + init_info.rank_offset
|
||||
# Create stateless process group
|
||||
device = torch.accelerator.current_device_index()
|
||||
self.model_update_group = (
|
||||
NCCLWeightTransferEngine._stateless_init_process_group(
|
||||
init_info.master_address,
|
||||
init_info.master_port,
|
||||
rank,
|
||||
init_info.world_size,
|
||||
device=device,
|
||||
)
|
||||
self.model_update_group = worker_init_process_group(
|
||||
init_info, self.parallel_config
|
||||
)
|
||||
|
||||
def receive_weights(
|
||||
self,
|
||||
update_info: NCCLWeightTransferUpdateInfo,
|
||||
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
|
||||
) -> None:
|
||||
def start_weight_update(self) -> None:
|
||||
"""Initialize layerwise reloading for the incoming checkpoint weights."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
|
||||
initialize_layerwise_reload(self.model)
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Finalize layerwise reloading after all weights have been received."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
)
|
||||
|
||||
finalize_layerwise_reload(self.model, self.model_config)
|
||||
|
||||
def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None:
|
||||
"""
|
||||
Receive weights from trainer via NCCL broadcast and load them incrementally.
|
||||
Receive weights from trainer via NCCL broadcast.
|
||||
|
||||
If update_info.packed is True, uses packed tensor broadcasting for
|
||||
efficient transfer of multiple weights in batches. Otherwise, uses simple
|
||||
@@ -179,19 +161,12 @@ class NCCLWeightTransferEngine(
|
||||
Args:
|
||||
update_info: NCCL update info containing parameter names, dtypes, shapes,
|
||||
and packed flag
|
||||
load_weights: Callable that loads weights into the model. Called
|
||||
incrementally for each batch of weights to avoid OOM.
|
||||
"""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"NCCL weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
if update_info.update_kind != "dense":
|
||||
raise ValueError(
|
||||
"Sparse updates must use `receive_sparse_weights`, not "
|
||||
"`receive_weights`"
|
||||
)
|
||||
|
||||
if update_info.packed:
|
||||
# Build iterator of (name, (shape, dtype)) from update_info
|
||||
@@ -206,9 +181,10 @@ class NCCLWeightTransferEngine(
|
||||
iterator=state_dict_info_iterator(),
|
||||
group=self.model_update_group,
|
||||
src=0,
|
||||
post_unpack_func=load_weights,
|
||||
post_unpack_func=self.model.load_weights,
|
||||
buffer_size_bytes=update_info.packed_buffer_size_bytes,
|
||||
num_buffers=update_info.packed_num_buffers,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
# Use simple one-by-one broadcasting
|
||||
@@ -216,49 +192,13 @@ class NCCLWeightTransferEngine(
|
||||
update_info.names, update_info.dtype_names, update_info.shapes
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
weight = torch.empty(shape, dtype=dtype, device="cuda")
|
||||
weight = torch.empty(shape, dtype=dtype, device=self.device)
|
||||
self.model_update_group.broadcast(
|
||||
weight, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
load_weights([(name, weight)])
|
||||
self.model.load_weights([(name, weight)])
|
||||
del weight
|
||||
|
||||
def receive_sparse_weights(
|
||||
self,
|
||||
update_info: NCCLWeightTransferUpdateInfo,
|
||||
apply_patches: Callable[[list[SparseWeightPatch]], None],
|
||||
) -> None:
|
||||
"""Receive sparse flat-index patches from trainer via NCCL."""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"NCCL weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
if update_info.update_kind != "sparse_flat":
|
||||
raise ValueError("Sparse receive path requires `update_kind='sparse_flat'`")
|
||||
assert update_info.num_updates_list is not None
|
||||
|
||||
for name, dtype_name, num_updates in zip(
|
||||
update_info.names,
|
||||
update_info.dtype_names,
|
||||
update_info.num_updates_list,
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
device = torch.accelerator.current_device_index()
|
||||
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
|
||||
values = torch.empty(num_updates, dtype=dtype, device=device)
|
||||
self.model_update_group.broadcast(
|
||||
indices, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
self.model_update_group.broadcast(
|
||||
values, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
apply_patches(
|
||||
[SparseWeightPatch(name=name, indices=indices, values=values)]
|
||||
)
|
||||
del indices
|
||||
del values
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
# Clean up the communicator by removing the reference
|
||||
@@ -269,7 +209,7 @@ class NCCLWeightTransferEngine(
|
||||
iterator: Iterator[tuple[str, torch.Tensor]],
|
||||
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
|
||||
) -> None:
|
||||
"""Broadcast weights from trainer to vLLM workers.
|
||||
"""Broadcast dense weights from trainer to vLLM workers.
|
||||
|
||||
Args:
|
||||
iterator: Iterator of model parameters. Returns (name, tensor) tuples
|
||||
@@ -322,94 +262,6 @@ class NCCLWeightTransferEngine(
|
||||
stream=args.stream or torch.cuda.current_stream(),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def trainer_send_sparse_weights(
|
||||
iterator: Iterator[SparseWeightPatch],
|
||||
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
|
||||
) -> None:
|
||||
"""Broadcast sparse flat-index patches from trainer to vLLM workers."""
|
||||
if isinstance(trainer_args, dict):
|
||||
args = NCCLTrainerSendWeightsArgs(**trainer_args)
|
||||
else:
|
||||
args = trainer_args
|
||||
|
||||
if args.packed:
|
||||
raise ValueError(
|
||||
"Sparse NCCL updates cannot be combined with `packed=True`"
|
||||
)
|
||||
|
||||
stream = args.stream or torch.cuda.current_stream()
|
||||
for patch in iterator:
|
||||
args.group.broadcast(patch.indices, src=args.src, stream=stream)
|
||||
args.group.broadcast(patch.values, src=args.src, stream=stream)
|
||||
|
||||
@staticmethod
|
||||
def trainer_init(
|
||||
init_info: NCCLWeightTransferInitInfo | dict,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""
|
||||
Initialize NCCL process group for trainer-side weight transfer.
|
||||
|
||||
The trainer is always rank 0 in the process group. Uses the current
|
||||
CUDA device (torch.accelerator.current_device_index()).
|
||||
|
||||
Args:
|
||||
init_info: Either an NCCLWeightTransferInitInfo object or a dict with keys:
|
||||
- master_address: str
|
||||
- master_port: int
|
||||
- world_size: int
|
||||
|
||||
Returns:
|
||||
PyNcclCommunicator for weight transfer.
|
||||
|
||||
Example:
|
||||
>>> from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
... NCCLWeightTransferEngine,
|
||||
... )
|
||||
>>> group = NCCLWeightTransferEngine.trainer_init(
|
||||
... dict(
|
||||
... master_address=master_address,
|
||||
... master_port=master_port,
|
||||
... world_size=world_size,
|
||||
... ),
|
||||
... )
|
||||
"""
|
||||
if isinstance(init_info, dict):
|
||||
master_address = init_info["master_address"]
|
||||
master_port = init_info["master_port"]
|
||||
world_size = init_info["world_size"]
|
||||
else:
|
||||
# NCCLWeightTransferInitInfo object
|
||||
master_address = init_info.master_address
|
||||
master_port = init_info.master_port
|
||||
world_size = init_info.world_size
|
||||
|
||||
# Trainer is always rank 0
|
||||
device = torch.accelerator.current_device_index()
|
||||
return NCCLWeightTransferEngine._stateless_init_process_group(
|
||||
master_address,
|
||||
master_port,
|
||||
0,
|
||||
world_size,
|
||||
device,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _stateless_init_process_group(
|
||||
master_address, master_port, rank, world_size, device
|
||||
):
|
||||
"""
|
||||
vLLM provides `StatelessProcessGroup` to create a process group
|
||||
without considering the global process group in torch.distributed.
|
||||
It is recommended to create `StatelessProcessGroup`, and then initialize
|
||||
the data-plane communication (NCCL) between external (train processes)
|
||||
and vLLM workers.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
pg = StatelessProcessGroup.create(
|
||||
host=master_address, port=master_port, rank=rank, world_size=world_size
|
||||
)
|
||||
pynccl = PyNcclCommunicator(pg, device=device)
|
||||
return pynccl
|
||||
# Trainer-side process-group setup. Delegates to the shared helper so the
|
||||
# sparse engine can reuse the exact same rendezvous without subclassing.
|
||||
trainer_init = staticmethod(trainer_init)
|
||||
|
||||
@@ -186,6 +186,7 @@ def packed_nccl_broadcast_consumer(
|
||||
post_unpack_func: Callable[[list[tuple[str, torch.Tensor]]], None],
|
||||
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
|
||||
device: torch.device | str = "cuda",
|
||||
) -> None:
|
||||
"""Consume packed tensors and unpack them into a list of tensors.
|
||||
|
||||
@@ -199,6 +200,8 @@ def packed_nccl_broadcast_consumer(
|
||||
Both producer and consumer must use the same value.
|
||||
num_buffers: Number of buffers for double/triple buffering.
|
||||
Both producer and consumer must use the same value.
|
||||
device: Device for the receive buffers. Must match the device the NCCL
|
||||
communicator was created on (the worker's assigned device).
|
||||
|
||||
"""
|
||||
target_packed_tensor_size = buffer_size_bytes
|
||||
@@ -211,7 +214,7 @@ def packed_nccl_broadcast_consumer(
|
||||
]
|
||||
packing_tensor_sizes: list[int] = [0 for _ in range(num_buffers)]
|
||||
packed_tensors: list[torch.Tensor] = [
|
||||
torch.empty(0, dtype=torch.uint8, device="cuda") for _ in range(num_buffers)
|
||||
torch.empty(0, dtype=torch.uint8, device=device) for _ in range(num_buffers)
|
||||
]
|
||||
|
||||
while True:
|
||||
@@ -234,7 +237,7 @@ def packed_nccl_broadcast_consumer(
|
||||
break
|
||||
# Create a packed tensor and broadcast it
|
||||
packed_tensors[buffer_idx] = torch.empty(
|
||||
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device="cuda"
|
||||
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device=device
|
||||
)
|
||||
group.broadcast(packed_tensors[buffer_idx], src=src)
|
||||
# Load the packed tensor into the model
|
||||
@@ -259,7 +262,7 @@ def packed_nccl_broadcast_consumer(
|
||||
packed_tensors[buffer_idx] = torch.empty(
|
||||
packing_tensor_sizes[buffer_idx],
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
device=device,
|
||||
)
|
||||
group.broadcast(packed_tensors[buffer_idx], src=src)
|
||||
# Load the packed tensor into the model
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sparse NCCL weight transfer engine.
|
||||
|
||||
A standalone engine (not a subclass of `NCCLWeightTransferEngine`) for applying
|
||||
sparse, flat-index weight patches in place. It shares only NCCL process-group
|
||||
initialization with the dense engine (via `nccl_common`); the update path
|
||||
applies index/value patches directly to existing model parameters and never runs
|
||||
layerwise reload.
|
||||
|
||||
MVP limitations:
|
||||
* TP=1 and PP=1 only
|
||||
* uses runtime/kernel-format parameter names
|
||||
* not composable with checkpoint-format or packed updates
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
WeightTransferEngine,
|
||||
WeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
trainer_init,
|
||||
worker_init_process_group,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLTrainerSendWeightsArgs
|
||||
|
||||
__all__ = [
|
||||
"SparseWeightPatch",
|
||||
"SparseNCCLWeightTransferUpdateInfo",
|
||||
"SparseNCCLWeightTransferEngine",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseWeightPatch:
|
||||
"""A sparse in-place patch for one existing parameter."""
|
||||
|
||||
name: str
|
||||
indices: torch.Tensor
|
||||
values: torch.Tensor
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Update info for the sparse NCCL weight transfer backend."""
|
||||
|
||||
names: list[str]
|
||||
dtype_names: list[str]
|
||||
shapes: list[list[int]]
|
||||
num_updates_list: list[int]
|
||||
"""Number of sparse entries to receive for each parameter in ``names``."""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
num_params = len(self.names)
|
||||
if len(self.dtype_names) != num_params:
|
||||
raise ValueError(
|
||||
f"`dtype_names` should be of the same size as `names`: "
|
||||
f"got {len(self.dtype_names)} and {len(self.names)}"
|
||||
)
|
||||
if len(self.shapes) != num_params:
|
||||
raise ValueError(
|
||||
f"`shapes` should be of the same size as `names`: "
|
||||
f"got {len(self.shapes)} and {len(self.names)}"
|
||||
)
|
||||
if len(self.num_updates_list) == 0:
|
||||
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
|
||||
if len(self.num_updates_list) != num_params:
|
||||
raise ValueError(
|
||||
f"`num_updates_list` should be of the same size as `names`: "
|
||||
f"got {len(self.num_updates_list)} and {len(self.names)}"
|
||||
)
|
||||
if any(num_updates < 0 for num_updates in self.num_updates_list):
|
||||
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
|
||||
|
||||
|
||||
class SparseNCCLWeightTransferEngine(
|
||||
WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo]
|
||||
):
|
||||
"""
|
||||
Sparse weight transfer engine using NCCL.
|
||||
|
||||
Receives flat-index (indices, values) patches broadcast from the trainer
|
||||
(rank 0) and applies them in place to existing model parameters. Weights are
|
||||
applied directly without layerwise reload, so `start_weight_update` and
|
||||
`finish_weight_update` are no-ops.
|
||||
"""
|
||||
|
||||
init_info_cls = NCCLWeightTransferInitInfo
|
||||
update_info_cls = SparseNCCLWeightTransferUpdateInfo
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, device, model)
|
||||
self.model_update_group: PyNcclCommunicator | None = None
|
||||
|
||||
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
|
||||
"""Initialize the NCCL process group with the trainer."""
|
||||
self.model_update_group = worker_init_process_group(
|
||||
init_info, self.parallel_config
|
||||
)
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""No-op: sparse patches are applied in place, no layerwise reload."""
|
||||
if self.parallel_config.world_size != 1:
|
||||
raise NotImplementedError(
|
||||
"Sparse weight updates currently require TP=1 and PP=1"
|
||||
)
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""No-op: sparse patches are applied in place, no layerwise reload."""
|
||||
pass
|
||||
|
||||
def receive_weights(self, update_info: SparseNCCLWeightTransferUpdateInfo) -> None:
|
||||
"""Receive sparse flat-index patches from the trainer and apply them."""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"NCCL weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
|
||||
# Use the worker's assigned device rather than the ambient current
|
||||
# device: the receive path is no longer wrapped in
|
||||
# `with torch.device(self.device)` by the caller, so the current device
|
||||
# is not guaranteed to match self.device. The recv buffers must live on
|
||||
# the same device as the NCCL communicator (created on self.device).
|
||||
device = self.device
|
||||
for name, dtype_name, num_updates in zip(
|
||||
update_info.names,
|
||||
update_info.dtype_names,
|
||||
update_info.num_updates_list,
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
|
||||
values = torch.empty(num_updates, dtype=dtype, device=device)
|
||||
self.model_update_group.broadcast(
|
||||
indices, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
self.model_update_group.broadcast(
|
||||
values, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
self._apply_patch(
|
||||
SparseWeightPatch(name=name, indices=indices, values=values)
|
||||
)
|
||||
del indices
|
||||
del values
|
||||
|
||||
def _apply_patch(self, patch: SparseWeightPatch) -> None:
|
||||
"""Apply a single sparse flat-index patch to an existing model param."""
|
||||
param = self.model.get_parameter(patch.name)
|
||||
if not param.data.is_contiguous():
|
||||
raise NotImplementedError(
|
||||
"Sparse weight updates currently require contiguous params: "
|
||||
f"{patch.name}"
|
||||
)
|
||||
if patch.indices.dtype != torch.int32:
|
||||
raise ValueError(
|
||||
f"Sparse weight updates currently require int32 indices: {patch.name}"
|
||||
)
|
||||
if patch.indices.ndim != 1 or patch.values.ndim != 1:
|
||||
raise ValueError(
|
||||
f"Sparse weight patches must be 1D flattened updates: {patch.name}"
|
||||
)
|
||||
if patch.indices.numel() != patch.values.numel():
|
||||
raise ValueError(
|
||||
f"`indices` and `values` must have matching lengths for {patch.name}"
|
||||
)
|
||||
if patch.values.dtype != param.dtype:
|
||||
raise ValueError(
|
||||
f"Sparse values dtype {patch.values.dtype} does not match "
|
||||
f"parameter dtype {param.dtype} for {patch.name}"
|
||||
)
|
||||
|
||||
flat_param = param.data.view(-1)
|
||||
flat_param.index_copy_(
|
||||
0,
|
||||
patch.indices.to(device=flat_param.device, dtype=torch.long),
|
||||
patch.values.to(device=flat_param.device),
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
self.model_update_group = None
|
||||
|
||||
@staticmethod
|
||||
def trainer_send_weights(
|
||||
iterator: Iterator[SparseWeightPatch],
|
||||
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
|
||||
) -> None:
|
||||
"""Broadcast sparse flat-index patches from trainer to vLLM workers."""
|
||||
if isinstance(trainer_args, dict):
|
||||
args = NCCLTrainerSendWeightsArgs(**trainer_args)
|
||||
else:
|
||||
args = trainer_args
|
||||
|
||||
if args.packed:
|
||||
raise ValueError(
|
||||
"Sparse NCCL updates cannot be combined with `packed=True`"
|
||||
)
|
||||
|
||||
stream = args.stream or torch.cuda.current_stream()
|
||||
for patch in iterator:
|
||||
args.group.broadcast(patch.indices, src=args.src, stream=stream)
|
||||
args.group.broadcast(patch.values, src=args.src, stream=stream)
|
||||
|
||||
# Trainer-side process-group setup (shared with the dense engine).
|
||||
trainer_init = staticmethod(trainer_init)
|
||||
Reference in New Issue
Block a user