[Frontend][Core] Add sparse NCCL weight transfer support for in-place updates (#40096)
Signed-off-by: Siddharth Bedekar <bedeksid@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
182c67daf1
commit
266b9d9c64
@@ -4,8 +4,8 @@
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Generic, TypeVar
|
||||
from dataclasses import KW_ONLY, dataclass, field
|
||||
from typing import Any, Generic, Literal, TypeVar
|
||||
|
||||
import torch
|
||||
|
||||
@@ -28,7 +28,44 @@ class WeightTransferInitInfo(ABC): # noqa: B024
|
||||
class WeightTransferUpdateInfo(ABC): # noqa: B024
|
||||
"""Base class for backend-specific weight update info."""
|
||||
|
||||
pass
|
||||
_: KW_ONLY
|
||||
update_kind: Literal["dense", "sparse_flat"] = "dense"
|
||||
"""Weight update format."""
|
||||
num_updates_list: list[int] | None = None
|
||||
"""Number of sparse entries to receive for each parameter in ``names``."""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.update_kind not in ("dense", "sparse_flat"):
|
||||
raise ValueError(f"Unsupported update_kind: {self.update_kind}")
|
||||
if self.update_kind == "dense":
|
||||
if self.num_updates_list is not None:
|
||||
raise ValueError(
|
||||
"Sparse metadata is only supported for `update_kind='sparse_flat'`"
|
||||
)
|
||||
return
|
||||
|
||||
if self.num_updates_list is None:
|
||||
raise ValueError("`num_updates_list` is required for sparse updates")
|
||||
if len(self.num_updates_list) == 0:
|
||||
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
|
||||
if any(num_updates < 0 for num_updates in self.num_updates_list):
|
||||
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
|
||||
|
||||
names = getattr(self, "names", None)
|
||||
if names is not None and len(self.num_updates_list) != len(names):
|
||||
raise ValueError(
|
||||
f"`num_updates_list` should be of the same size as `names`: "
|
||||
f"got {len(self.num_updates_list)} and {len(names)}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparseWeightPatch:
|
||||
"""A sparse in-place patch for one existing parameter."""
|
||||
|
||||
name: str
|
||||
indices: torch.Tensor
|
||||
values: torch.Tensor
|
||||
|
||||
|
||||
# API-level request classes (accept dicts for backend-agnostic serialization)
|
||||
@@ -150,6 +187,16 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def receive_sparse_weights(
|
||||
self,
|
||||
update_info: TUpdateInfo,
|
||||
apply_patches: Callable[[list[SparseWeightPatch]], None],
|
||||
) -> None:
|
||||
"""Receive sparse weight patches from the trainer."""
|
||||
raise NotImplementedError(
|
||||
f"{self.__class__.__name__} does not support sparse weight updates"
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def shutdown(self) -> None:
|
||||
"""
|
||||
@@ -184,3 +231,11 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
>>> engine.trainer_send_weights(param_iter, trainer_args)
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def trainer_send_sparse_weights(
|
||||
_iterator: Iterator[SparseWeightPatch],
|
||||
_trainer_args: dict[str, Any] | Any,
|
||||
) -> None:
|
||||
"""Send sparse weight patches from trainer to inference workers."""
|
||||
raise NotImplementedError("Sparse weight updates are not supported")
|
||||
|
||||
@@ -74,10 +74,12 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
names: list[str]
|
||||
dtype_names: list[str]
|
||||
shapes: list[list[int]]
|
||||
ipc_handles: list[dict[str, tuple]] | dict[str, tuple]
|
||||
ipc_handles: list[dict[str, tuple]] | dict[str, tuple] | None = None
|
||||
"""IPC handles mapping physical GPU UUID to rebuild_cuda_tensor args.
|
||||
For non-packed mode: list of per-parameter handle dicts.
|
||||
For packed mode: single handle dict for the packed buffer."""
|
||||
ipc_handles_pickled: str | None = None
|
||||
"""Base64-encoded pickled IPC handles, used for HTTP transport."""
|
||||
tensor_sizes: list[int] | None = None
|
||||
"""Per-parameter sizes in bytes within the packed buffer.
|
||||
Required when packed=True, unused otherwise."""
|
||||
@@ -85,6 +87,29 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
"""Whether this update uses packed tensor format."""
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
if self.update_kind != "dense":
|
||||
raise NotImplementedError("IPC weight transfer only supports dense updates")
|
||||
|
||||
if self.ipc_handles_pickled is not None:
|
||||
if self.ipc_handles is not None:
|
||||
raise ValueError(
|
||||
"Cannot specify both `ipc_handles` and `ipc_handles_pickled`"
|
||||
)
|
||||
|
||||
if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION:
|
||||
raise ValueError(
|
||||
"Refusing to deserialize `ipc_handles_pickled` without "
|
||||
"VLLM_ALLOW_INSECURE_SERIALIZATION=1"
|
||||
)
|
||||
|
||||
self.ipc_handles = pickle.loads(base64.b64decode(self.ipc_handles_pickled))
|
||||
self.ipc_handles_pickled = None
|
||||
|
||||
if self.ipc_handles is None:
|
||||
raise ValueError(
|
||||
"Either `ipc_handles` or `ipc_handles_pickled` must be provided"
|
||||
)
|
||||
num_params = len(self.names)
|
||||
if len(self.dtype_names) != num_params:
|
||||
raise ValueError(
|
||||
@@ -153,8 +178,9 @@ class IPCWeightTransferEngine(
|
||||
Requires ``VLLM_ALLOW_INSECURE_SERIALIZATION=1`` because the
|
||||
payload is deserialized via ``pickle.loads``.
|
||||
"""
|
||||
if "ipc_handles_pickled" in update_dict:
|
||||
if "ipc_handles" in update_dict:
|
||||
pickled = update_dict.pop("ipc_handles_pickled", None)
|
||||
if pickled is not None:
|
||||
if update_dict.get("ipc_handles") is not None:
|
||||
raise ValueError(
|
||||
"Cannot specify both `ipc_handles` and `ipc_handles_pickled`"
|
||||
)
|
||||
@@ -165,7 +191,6 @@ class IPCWeightTransferEngine(
|
||||
"VLLM_ALLOW_INSECURE_SERIALIZATION=1"
|
||||
)
|
||||
|
||||
pickled = update_dict.pop("ipc_handles_pickled")
|
||||
update_dict["ipc_handles"] = pickle.loads(base64.b64decode(pickled))
|
||||
|
||||
return super().parse_update_info(update_dict)
|
||||
|
||||
@@ -14,6 +14,7 @@ if TYPE_CHECKING:
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
SparseWeightPatch,
|
||||
WeightTransferEngine,
|
||||
WeightTransferInitInfo,
|
||||
WeightTransferUpdateInfo,
|
||||
@@ -81,6 +82,7 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
|
||||
def __post_init__(self):
|
||||
"""Validate that all lists have the same length."""
|
||||
super().__post_init__()
|
||||
num_params = len(self.names)
|
||||
if len(self.dtype_names) != num_params:
|
||||
raise ValueError(
|
||||
@@ -92,6 +94,13 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
f"`shapes` should be of the same size as `names`: "
|
||||
f"got {len(self.shapes)} and {len(self.names)}"
|
||||
)
|
||||
if self.update_kind == "dense":
|
||||
return
|
||||
|
||||
if self.packed:
|
||||
raise ValueError(
|
||||
"`update_kind='sparse_flat'` cannot be combined with `packed=True`"
|
||||
)
|
||||
|
||||
|
||||
class NCCLWeightTransferEngine(
|
||||
@@ -178,6 +187,11 @@ class NCCLWeightTransferEngine(
|
||||
"NCCL weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
if update_info.update_kind != "dense":
|
||||
raise ValueError(
|
||||
"Sparse updates must use `receive_sparse_weights`, not "
|
||||
"`receive_weights`"
|
||||
)
|
||||
|
||||
if update_info.packed:
|
||||
# Build iterator of (name, (shape, dtype)) from update_info
|
||||
@@ -209,6 +223,42 @@ class NCCLWeightTransferEngine(
|
||||
load_weights([(name, weight)])
|
||||
del weight
|
||||
|
||||
def receive_sparse_weights(
|
||||
self,
|
||||
update_info: NCCLWeightTransferUpdateInfo,
|
||||
apply_patches: Callable[[list[SparseWeightPatch]], None],
|
||||
) -> None:
|
||||
"""Receive sparse flat-index patches from trainer via NCCL."""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
"NCCL weight transfer not initialized. "
|
||||
"Call init_transfer_engine() first."
|
||||
)
|
||||
if update_info.update_kind != "sparse_flat":
|
||||
raise ValueError("Sparse receive path requires `update_kind='sparse_flat'`")
|
||||
assert update_info.num_updates_list is not None
|
||||
|
||||
for name, dtype_name, num_updates in zip(
|
||||
update_info.names,
|
||||
update_info.dtype_names,
|
||||
update_info.num_updates_list,
|
||||
):
|
||||
dtype = getattr(torch, dtype_name)
|
||||
device = torch.accelerator.current_device_index()
|
||||
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
|
||||
values = torch.empty(num_updates, dtype=dtype, device=device)
|
||||
self.model_update_group.broadcast(
|
||||
indices, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
self.model_update_group.broadcast(
|
||||
values, src=0, stream=torch.cuda.current_stream()
|
||||
)
|
||||
apply_patches(
|
||||
[SparseWeightPatch(name=name, indices=indices, values=values)]
|
||||
)
|
||||
del indices
|
||||
del values
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.model_update_group is not None:
|
||||
# Clean up the communicator by removing the reference
|
||||
@@ -272,6 +322,27 @@ class NCCLWeightTransferEngine(
|
||||
stream=args.stream or torch.cuda.current_stream(),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def trainer_send_sparse_weights(
|
||||
iterator: Iterator[SparseWeightPatch],
|
||||
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
|
||||
) -> None:
|
||||
"""Broadcast sparse flat-index patches from trainer to vLLM workers."""
|
||||
if isinstance(trainer_args, dict):
|
||||
args = NCCLTrainerSendWeightsArgs(**trainer_args)
|
||||
else:
|
||||
args = trainer_args
|
||||
|
||||
if args.packed:
|
||||
raise ValueError(
|
||||
"Sparse NCCL updates cannot be combined with `packed=True`"
|
||||
)
|
||||
|
||||
stream = args.stream or torch.cuda.current_stream()
|
||||
for patch in iterator:
|
||||
args.group.broadcast(patch.indices, src=args.src, stream=stream)
|
||||
args.group.broadcast(patch.values, src=args.src, stream=stream)
|
||||
|
||||
@staticmethod
|
||||
def trainer_init(
|
||||
init_info: NCCLWeightTransferInitInfo | dict,
|
||||
|
||||
Reference in New Issue
Block a user