[Frontend][Core] Add sparse NCCL weight transfer support for in-place updates (#40096)

Signed-off-by: Siddharth Bedekar <bedeksid@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Siddharth Bedekar
2026-06-01 15:37:30 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent 182c67daf1
commit 266b9d9c64
12 changed files with 1423 additions and 75 deletions
+58 -3
View File
@@ -4,8 +4,8 @@
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterator
from dataclasses import dataclass, field
from typing import Any, Generic, TypeVar
from dataclasses import KW_ONLY, dataclass, field
from typing import Any, Generic, Literal, TypeVar
import torch
@@ -28,7 +28,44 @@ class WeightTransferInitInfo(ABC): # noqa: B024
class WeightTransferUpdateInfo(ABC): # noqa: B024
"""Base class for backend-specific weight update info."""
pass
_: KW_ONLY
update_kind: Literal["dense", "sparse_flat"] = "dense"
"""Weight update format."""
num_updates_list: list[int] | None = None
"""Number of sparse entries to receive for each parameter in ``names``."""
def __post_init__(self) -> None:
if self.update_kind not in ("dense", "sparse_flat"):
raise ValueError(f"Unsupported update_kind: {self.update_kind}")
if self.update_kind == "dense":
if self.num_updates_list is not None:
raise ValueError(
"Sparse metadata is only supported for `update_kind='sparse_flat'`"
)
return
if self.num_updates_list is None:
raise ValueError("`num_updates_list` is required for sparse updates")
if len(self.num_updates_list) == 0:
raise ValueError("`num_updates_list` cannot be empty for sparse updates")
if any(num_updates < 0 for num_updates in self.num_updates_list):
raise ValueError("Sparse `num_updates_list` entries must be non-negative")
names = getattr(self, "names", None)
if names is not None and len(self.num_updates_list) != len(names):
raise ValueError(
f"`num_updates_list` should be of the same size as `names`: "
f"got {len(self.num_updates_list)} and {len(names)}"
)
@dataclass
class SparseWeightPatch:
"""A sparse in-place patch for one existing parameter."""
name: str
indices: torch.Tensor
values: torch.Tensor
# API-level request classes (accept dicts for backend-agnostic serialization)
@@ -150,6 +187,16 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
"""
raise NotImplementedError
def receive_sparse_weights(
self,
update_info: TUpdateInfo,
apply_patches: Callable[[list[SparseWeightPatch]], None],
) -> None:
"""Receive sparse weight patches from the trainer."""
raise NotImplementedError(
f"{self.__class__.__name__} does not support sparse weight updates"
)
@abstractmethod
def shutdown(self) -> None:
"""
@@ -184,3 +231,11 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
>>> engine.trainer_send_weights(param_iter, trainer_args)
"""
raise NotImplementedError
@staticmethod
def trainer_send_sparse_weights(
_iterator: Iterator[SparseWeightPatch],
_trainer_args: dict[str, Any] | Any,
) -> None:
"""Send sparse weight patches from trainer to inference workers."""
raise NotImplementedError("Sparse weight updates are not supported")
+29 -4
View File
@@ -74,10 +74,12 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
names: list[str]
dtype_names: list[str]
shapes: list[list[int]]
ipc_handles: list[dict[str, tuple]] | dict[str, tuple]
ipc_handles: list[dict[str, tuple]] | dict[str, tuple] | None = None
"""IPC handles mapping physical GPU UUID to rebuild_cuda_tensor args.
For non-packed mode: list of per-parameter handle dicts.
For packed mode: single handle dict for the packed buffer."""
ipc_handles_pickled: str | None = None
"""Base64-encoded pickled IPC handles, used for HTTP transport."""
tensor_sizes: list[int] | None = None
"""Per-parameter sizes in bytes within the packed buffer.
Required when packed=True, unused otherwise."""
@@ -85,6 +87,29 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Whether this update uses packed tensor format."""
def __post_init__(self):
super().__post_init__()
if self.update_kind != "dense":
raise NotImplementedError("IPC weight transfer only supports dense updates")
if self.ipc_handles_pickled is not None:
if self.ipc_handles is not None:
raise ValueError(
"Cannot specify both `ipc_handles` and `ipc_handles_pickled`"
)
if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION:
raise ValueError(
"Refusing to deserialize `ipc_handles_pickled` without "
"VLLM_ALLOW_INSECURE_SERIALIZATION=1"
)
self.ipc_handles = pickle.loads(base64.b64decode(self.ipc_handles_pickled))
self.ipc_handles_pickled = None
if self.ipc_handles is None:
raise ValueError(
"Either `ipc_handles` or `ipc_handles_pickled` must be provided"
)
num_params = len(self.names)
if len(self.dtype_names) != num_params:
raise ValueError(
@@ -153,8 +178,9 @@ class IPCWeightTransferEngine(
Requires ``VLLM_ALLOW_INSECURE_SERIALIZATION=1`` because the
payload is deserialized via ``pickle.loads``.
"""
if "ipc_handles_pickled" in update_dict:
if "ipc_handles" in update_dict:
pickled = update_dict.pop("ipc_handles_pickled", None)
if pickled is not None:
if update_dict.get("ipc_handles") is not None:
raise ValueError(
"Cannot specify both `ipc_handles` and `ipc_handles_pickled`"
)
@@ -165,7 +191,6 @@ class IPCWeightTransferEngine(
"VLLM_ALLOW_INSECURE_SERIALIZATION=1"
)
pickled = update_dict.pop("ipc_handles_pickled")
update_dict["ipc_handles"] = pickle.loads(base64.b64decode(pickled))
return super().parse_update_info(update_dict)
@@ -14,6 +14,7 @@ if TYPE_CHECKING:
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
SparseWeightPatch,
WeightTransferEngine,
WeightTransferInitInfo,
WeightTransferUpdateInfo,
@@ -81,6 +82,7 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
def __post_init__(self):
"""Validate that all lists have the same length."""
super().__post_init__()
num_params = len(self.names)
if len(self.dtype_names) != num_params:
raise ValueError(
@@ -92,6 +94,13 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
f"`shapes` should be of the same size as `names`: "
f"got {len(self.shapes)} and {len(self.names)}"
)
if self.update_kind == "dense":
return
if self.packed:
raise ValueError(
"`update_kind='sparse_flat'` cannot be combined with `packed=True`"
)
class NCCLWeightTransferEngine(
@@ -178,6 +187,11 @@ class NCCLWeightTransferEngine(
"NCCL weight transfer not initialized. "
"Call init_transfer_engine() first."
)
if update_info.update_kind != "dense":
raise ValueError(
"Sparse updates must use `receive_sparse_weights`, not "
"`receive_weights`"
)
if update_info.packed:
# Build iterator of (name, (shape, dtype)) from update_info
@@ -209,6 +223,42 @@ class NCCLWeightTransferEngine(
load_weights([(name, weight)])
del weight
def receive_sparse_weights(
self,
update_info: NCCLWeightTransferUpdateInfo,
apply_patches: Callable[[list[SparseWeightPatch]], None],
) -> None:
"""Receive sparse flat-index patches from trainer via NCCL."""
if self.model_update_group is None:
raise RuntimeError(
"NCCL weight transfer not initialized. "
"Call init_transfer_engine() first."
)
if update_info.update_kind != "sparse_flat":
raise ValueError("Sparse receive path requires `update_kind='sparse_flat'`")
assert update_info.num_updates_list is not None
for name, dtype_name, num_updates in zip(
update_info.names,
update_info.dtype_names,
update_info.num_updates_list,
):
dtype = getattr(torch, dtype_name)
device = torch.accelerator.current_device_index()
indices = torch.empty(num_updates, dtype=torch.int32, device=device)
values = torch.empty(num_updates, dtype=dtype, device=device)
self.model_update_group.broadcast(
indices, src=0, stream=torch.cuda.current_stream()
)
self.model_update_group.broadcast(
values, src=0, stream=torch.cuda.current_stream()
)
apply_patches(
[SparseWeightPatch(name=name, indices=indices, values=values)]
)
del indices
del values
def shutdown(self) -> None:
if self.model_update_group is not None:
# Clean up the communicator by removing the reference
@@ -272,6 +322,27 @@ class NCCLWeightTransferEngine(
stream=args.stream or torch.cuda.current_stream(),
)
@staticmethod
def trainer_send_sparse_weights(
iterator: Iterator[SparseWeightPatch],
trainer_args: dict[str, Any] | NCCLTrainerSendWeightsArgs,
) -> None:
"""Broadcast sparse flat-index patches from trainer to vLLM workers."""
if isinstance(trainer_args, dict):
args = NCCLTrainerSendWeightsArgs(**trainer_args)
else:
args = trainer_args
if args.packed:
raise ValueError(
"Sparse NCCL updates cannot be combined with `packed=True`"
)
stream = args.stream or torch.cuda.current_stream()
for patch in iterator:
args.group.broadcast(patch.indices, src=args.src, stream=stream)
args.group.broadcast(patch.values, src=args.src, stream=stream)
@staticmethod
def trainer_init(
init_info: NCCLWeightTransferInitInfo | dict,