[Feat][RL] IPC weight sync optimizations: multigpu support and chunked packed tensors (#37476)

Signed-off-by: ahao-anyscale <ahao@anyscale.com>
Signed-off-by: hao-aaron <ahao@anyscale.com>
This commit is contained in:
Aaron Hao
2026-05-15 22:53:06 +08:00
committed by GitHub
parent 0fe7550254
commit e0a45f1455
10 changed files with 1591 additions and 455 deletions
+291 -142
View File
@@ -8,9 +8,10 @@ from dataclasses import asdict, dataclass
from typing import Any
import pybase64 as base64
import ray
import requests
import torch
from torch.multiprocessing.reductions import reduce_tensor
from torch.multiprocessing.reductions import rebuild_cuda_tensor, reduce_tensor
from vllm import envs
from vllm.config.parallel import ParallelConfig
@@ -20,27 +21,43 @@ from vllm.distributed.weight_transfer.base import (
WeightTransferInitInfo,
WeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
packed_ipc_consumer,
packed_ipc_producer,
)
@dataclass
class IPCTrainerSendWeightsArgs:
"""Arguments for IPC trainer_send_weights method."""
mode: str
"""Transport mode: 'http' or 'ray'."""
send_mode: str | Callable[["IPCWeightTransferUpdateInfo"], None]
"""How to send updates to vLLM. Either a string ('ray' or 'http') for
built-in transports, or a callable that receives an
IPCWeightTransferUpdateInfo and performs the send."""
llm_handle: Any = None
"""Ray ObjectRef to LLM handle (required for 'ray' mode)."""
"""Ray actor handle or list of handles (required for 'ray' send_mode)."""
url: str | None = None
"""Base URL for HTTP endpoint (required for 'http' mode)."""
"""Base URL for HTTP endpoint (required for 'http' send_mode)."""
packed: bool = False
"""Whether to use packed tensor transfer for bounded-memory chunking."""
packed_buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES
"""Size in bytes for each packed tensor buffer when packed=True."""
def __post_init__(self):
"""Validate that required arguments are provided for the selected mode."""
if self.mode == "ray" and self.llm_handle is None:
raise ValueError("llm_handle is required for 'ray' mode")
if self.mode == "http" and self.url is None:
raise ValueError("url is required for 'http' mode")
if self.mode not in ("ray", "http"):
raise ValueError(f"mode must be 'ray' or 'http', got {self.mode}")
if callable(self.send_mode):
return
if self.send_mode == "ray" and self.llm_handle is None:
raise ValueError("llm_handle is required for 'ray' send_mode")
if self.send_mode == "http" and self.url is None:
raise ValueError("url is required for 'http' send_mode")
if self.send_mode not in ("ray", "http"):
raise ValueError(
f"send_mode must be 'ray', 'http', or a callable, "
f"got {self.send_mode!r}"
)
@dataclass
@@ -52,44 +69,22 @@ class IPCWeightTransferInitInfo(WeightTransferInitInfo):
@dataclass
class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
"""Update info for IPC weight transfer backend.
Accepts IPC handles either directly via ``ipc_handles`` (Ray transport)
or as a base64-encoded pickle via ``ipc_handles_pickled`` (HTTP transport).
Exactly one of the two must be provided; if ``ipc_handles_pickled`` is set
it is unpickled into ``ipc_handles`` during ``__post_init__``.
"""
"""Update info for IPC weight transfer backend."""
names: list[str]
dtype_names: list[str]
shapes: list[list[int]]
ipc_handles: list[dict[str, tuple[Callable, tuple]]] | None = None
"""IPC handles mapping physical GPU UUID to (func, args) tuple.
Each handle is a dictionary mapping GPU UUID strings to IPC handle tuples."""
ipc_handles_pickled: str | None = None
"""Base64-encoded pickled IPC handles, used for HTTP transport."""
ipc_handles: list[dict[str, tuple]] | dict[str, tuple]
"""IPC handles mapping physical GPU UUID to rebuild_cuda_tensor args.
For non-packed mode: list of per-parameter handle dicts.
For packed mode: single handle dict for the packed buffer."""
tensor_sizes: list[int] | None = None
"""Per-parameter sizes in bytes within the packed buffer.
Required when packed=True, unused otherwise."""
packed: bool = False
"""Whether this update uses packed tensor format."""
def __post_init__(self):
if self.ipc_handles_pickled is not None:
if self.ipc_handles is not None:
raise ValueError(
"Cannot specify both `ipc_handles` and `ipc_handles_pickled`"
)
if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION:
raise ValueError(
"Refusing to deserialize `ipc_handles_pickled` without "
"VLLM_ALLOW_INSECURE_SERIALIZATION=1"
)
self.ipc_handles = pickle.loads(base64.b64decode(self.ipc_handles_pickled))
self.ipc_handles_pickled = None
if self.ipc_handles is None:
raise ValueError(
"Either `ipc_handles` or `ipc_handles_pickled` must be provided"
)
num_params = len(self.names)
if len(self.dtype_names) != num_params:
raise ValueError(
@@ -101,11 +96,17 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
f"`shapes` should be of the same size as `names`: "
f"got {len(self.shapes)} and {len(self.names)}"
)
if len(self.ipc_handles) != num_params:
if (
not self.packed
and isinstance(self.ipc_handles, list)
and len(self.ipc_handles) != num_params
):
raise ValueError(
f"`ipc_handles` should be of the same size as `names`: "
f"got {len(self.ipc_handles)} and {len(self.names)}"
)
if self.packed and self.tensor_sizes is None:
raise ValueError("`tensor_sizes` is required when packed=True")
class IPCWeightTransferEngine(
@@ -135,6 +136,36 @@ class IPCWeightTransferEngine(
"""
super().__init__(config, parallel_config)
def parse_update_info(
self, update_dict: dict[str, Any]
) -> IPCWeightTransferUpdateInfo:
"""Parse update dict, deserializing pickled IPC handles if present.
HTTP transport sends IPC handles as a base64-encoded pickle under the
key ``ipc_handles_pickled``. This method deserializes them back into
``ipc_handles`` before constructing the typed dataclass, keeping
serialization concerns out of the dataclass itself.
Requires ``VLLM_ALLOW_INSECURE_SERIALIZATION=1`` because the
payload is deserialized via ``pickle.loads``.
"""
if "ipc_handles_pickled" in update_dict:
if "ipc_handles" in update_dict:
raise ValueError(
"Cannot specify both `ipc_handles` and `ipc_handles_pickled`"
)
if not envs.VLLM_ALLOW_INSECURE_SERIALIZATION:
raise ValueError(
"Refusing to deserialize `ipc_handles_pickled` without "
"VLLM_ALLOW_INSECURE_SERIALIZATION=1"
)
pickled = update_dict.pop("ipc_handles_pickled")
update_dict["ipc_handles"] = pickle.loads(base64.b64decode(pickled))
return super().parse_update_info(update_dict)
def init_transfer_engine(self, init_info: IPCWeightTransferInitInfo) -> None:
"""
Initialize the weight transfer mechanism.
@@ -157,46 +188,52 @@ class IPCWeightTransferEngine(
Args:
update_info: IPC update info containing parameter names, dtypes, shapes,
and IPC handles. Each IPC handle is a mapping between physical
GPU UUID and the IPC handle tuple (func, args).
GPU UUID and the rebuild_cuda_tensor args tuple.
load_weights: Callable that loads weights into the model. Called
incrementally for each weight to avoid OOM.
"""
assert update_info.ipc_handles is not None
weights = []
for name, _dtype_name, _shape, ipc_handle in zip(
update_info.names,
update_info.dtype_names,
update_info.shapes,
update_info.ipc_handles,
):
device_index = torch.accelerator.current_device_index()
props = torch.cuda.get_device_properties(device_index)
physical_gpu_id = str(props.uuid)
device_index = torch.accelerator.current_device_index()
if physical_gpu_id not in ipc_handle:
raise ValueError(
f"IPC handle not found for GPU UUID {physical_gpu_id}. "
f"Available UUIDs: {list(ipc_handle.keys())}"
)
if update_info.packed:
assert update_info.tensor_sizes is not None
assert isinstance(update_info.ipc_handles, dict)
weights = packed_ipc_consumer(
ipc_handle=update_info.ipc_handles,
names=update_info.names,
shapes=update_info.shapes,
dtype_names=update_info.dtype_names,
tensor_sizes=update_info.tensor_sizes,
device_index=device_index,
)
load_weights(weights)
else:
assert isinstance(update_info.ipc_handles, list)
weights = []
for name, ipc_handle in zip(
update_info.names,
update_info.ipc_handles,
):
props = torch.cuda.get_device_properties(device_index)
physical_gpu_id = str(props.uuid)
handle = ipc_handle[physical_gpu_id]
if physical_gpu_id not in ipc_handle:
raise ValueError(
f"IPC handle not found for GPU UUID "
f"{physical_gpu_id}. "
f"Available UUIDs: {list(ipc_handle.keys())}"
)
func, args = handle
list_args = list(args) # type: ignore
# Index 6 is the device_index parameter in torch's
# IPC handle tuple (rebuild_cuda_tensor). Update it
# to the current device since the logical index can
# differ between sender and receiver.
list_args[6] = device_index
weight = func(*list_args) # type: ignore
weights.append((name, weight))
args = ipc_handle[physical_gpu_id]
list_args = list(args)
# Index 6 of the args from reduce_tensor is the device_index.
# We need to overwrite it with the receiver's device index.
list_args[6] = device_index
weight = rebuild_cuda_tensor(*list_args)
weights.append((name, weight))
load_weights(weights)
load_weights(weights)
def shutdown(self) -> None:
"""
Shutdown the weight transfer engine.
"""
pass
@staticmethod
@@ -204,12 +241,17 @@ class IPCWeightTransferEngine(
iterator: Iterator[tuple[str, torch.Tensor]],
trainer_args: dict[str, Any] | IPCTrainerSendWeightsArgs,
) -> None:
"""
Send weights from trainer to inference workers via CUDA IPC.
"""Send weights from trainer to inference workers via CUDA IPC.
Supports two modes:
- 'ray': Sends weights via Ray RPC to a Ray-based LLM handle
- 'http': Sends weights via HTTP POST to a vLLM HTTP server
Supports two transport modes ('ray' and 'http') and two transfer
strategies:
- Non-packed (default): all weights in a single API call.
- Packed (packed=True): chunked transfer with bounded GPU memory.
For multi-GPU training, all ranks must call this method in
parallel. IPC handles are all-gathered across ranks and merged
so that each vLLM worker can find its own GPU UUID. Only rank 0
sends the payload to vLLM.
.. note::
This method calls ``update_weights`` internally. The caller must
@@ -217,88 +259,195 @@ class IPCWeightTransferEngine(
after this method.
Args:
iterator: Iterator of model parameters. Returns (name, tensor) tuples.
Tensors should be on the same GPU as the inference workers.
trainer_args: Dictionary containing IPC-specific arguments.
Should contain keys from IPCTrainerSendWeightsArgs:
- mode: 'ray' or 'http'
- llm_handle: Ray ObjectRef (for 'ray' mode)
- url: Base URL string (for 'http' mode)
Example (Ray mode):
>>> from vllm.distributed.weight_transfer.ipc_engine import (
... IPCWeightTransferEngine,
... IPCTrainerSendWeightsArgs,
... )
>>> param_iter = ((n, p) for n, p in model.named_parameters())
>>> args = IPCTrainerSendWeightsArgs(mode="ray", llm_handle=llm_handle)
>>> IPCWeightTransferEngine.trainer_send_weights(param_iter, asdict(args))
Example (HTTP mode):
>>> args = IPCTrainerSendWeightsArgs(
... mode="http", url="http://localhost:8000"
... )
>>> IPCWeightTransferEngine.trainer_send_weights(param_iter, asdict(args))
iterator: Iterator of (name, tensor) pairs. For multi-GPU,
each rank should yield the full tensor on its own GPU
(e.g. via FSDP full_tensor()).
trainer_args: IPCTrainerSendWeightsArgs or equivalent dict.
"""
# Parse trainer args - accept either dict or dataclass instance
if isinstance(trainer_args, dict):
args = IPCTrainerSendWeightsArgs(**trainer_args)
else:
args = trainer_args
# Get physical GPU UUID
args = (
IPCTrainerSendWeightsArgs(**trainer_args)
if isinstance(trainer_args, dict)
else trainer_args
)
device_index = torch.accelerator.current_device_index()
props = torch.cuda.get_device_properties(device_index)
gpu_uuid = str(props.uuid)
gpu_uuid = str(torch.cuda.get_device_properties(device_index).uuid)
if args.packed:
IPCWeightTransferEngine._send_packed(iterator, args, gpu_uuid)
else:
IPCWeightTransferEngine._send_unpacked(iterator, args, gpu_uuid)
# Collect weight metadata and create IPC handles
names = []
dtype_names = []
shapes = []
ipc_handles = []
@staticmethod
def _is_rank_zero() -> bool:
"""Return True if this is rank 0 or no distributed group exists."""
if not torch.distributed.is_initialized():
return True
return torch.distributed.get_rank() == 0
@staticmethod
def _all_gather_and_merge_handles(
handles: list[dict[str, tuple]],
) -> list[dict[str, tuple]]:
"""All-gather and merge IPC handle dicts across ranks in one call.
Each rank contributes a list of {gpu_uuid: ipc_args} dicts (one
per parameter or one per chunk). A single all_gather_object
collects every rank's full list, then rank 0 merges per-index so
each dict maps every GPU UUID to its args.
Non-rank-0 returns a list of empty dicts.
No-op (returns handles unchanged) when no distributed group exists.
"""
if (
not torch.distributed.is_initialized()
or torch.distributed.get_world_size() == 1
):
return handles
world_size = torch.distributed.get_world_size()
gathered: list[list[dict[str, tuple]] | None] = [None] * world_size
torch.distributed.all_gather_object(gathered, handles)
torch.distributed.barrier()
torch.cuda.synchronize()
if torch.distributed.get_rank() == 0:
merged: list[dict[str, tuple]] = []
for param_idx in range(len(handles)):
m: dict[str, tuple] = {}
for rank_handles in gathered:
if rank_handles is not None:
m.update(rank_handles[param_idx])
merged.append(m)
return merged
return [{} for _ in handles]
@staticmethod
def _post_send_sync() -> None:
"""Barrier + ipc_collect after a send; no-op if single-GPU."""
if (
torch.distributed.is_initialized()
and torch.distributed.get_world_size() > 1
):
torch.distributed.barrier()
torch.cuda.ipc_collect()
@staticmethod
def _send_unpacked(
iterator: Iterator[tuple[str, torch.Tensor]],
args: IPCTrainerSendWeightsArgs,
gpu_uuid: str,
) -> None:
"""Send all weights in a single API call (non-packed mode)."""
names: list[str] = []
dtype_names: list[str] = []
shapes: list[list[int]] = []
ipc_handles: list[dict[str, tuple]] = []
# Hold strong refs to every contiguous copy until the send + post-send
# sync completes. reduce_tensor's returned args do NOT keep storage
# alive, and non-contiguous inputs allocate fresh storage in
# .contiguous() that would otherwise be GC'd before the consumer opens
# the IPC handle.
weight_refs: list[torch.Tensor] = []
for name, tensor in iterator:
names.append(name)
dtype_names.append(str(tensor.dtype).split(".")[-1])
shapes.append(list(tensor.shape))
# Create IPC handle for this weight tensor
# The tensor must remain in memory for IPC to work
weight = tensor.detach().contiguous()
ipc_handle = reduce_tensor(weight)
ipc_handles.append({gpu_uuid: ipc_handle})
weight_refs.append(weight)
_, ipc_args = reduce_tensor(weight)
ipc_handles.append({gpu_uuid: ipc_args})
# Send weights based on mode
if args.mode == "ray":
# Ray mode: send via Ray RPC
import ray
ipc_handles = IPCWeightTransferEngine._all_gather_and_merge_handles(ipc_handles)
update_info = asdict(
IPCWeightTransferUpdateInfo(
names=names,
dtype_names=dtype_names,
shapes=shapes,
ipc_handles=ipc_handles,
if IPCWeightTransferEngine._is_rank_zero():
IPCWeightTransferEngine._do_send(
args=args,
names=names,
dtype_names=dtype_names,
shapes=shapes,
ipc_handles=ipc_handles,
)
IPCWeightTransferEngine._post_send_sync()
@staticmethod
def _send_packed(
iterator: Iterator[tuple[str, torch.Tensor]],
args: IPCTrainerSendWeightsArgs,
gpu_uuid: str,
) -> None:
"""Send weights in bounded-memory chunks (packed mode)."""
post_iter_func: Callable = lambda item: item[1]
for chunk in packed_ipc_producer(
iterator=iterator,
gpu_uuid=gpu_uuid,
post_iter_func=post_iter_func,
buffer_size_bytes=args.packed_buffer_size_bytes,
):
ipc_handle = IPCWeightTransferEngine._all_gather_and_merge_handles(
[chunk.ipc_handle]
)[0]
if IPCWeightTransferEngine._is_rank_zero():
IPCWeightTransferEngine._do_send(
args=args,
names=chunk.names,
dtype_names=chunk.dtype_names,
shapes=chunk.shapes,
ipc_handles=ipc_handle,
tensor_sizes=chunk.tensor_sizes,
packed=True,
)
IPCWeightTransferEngine._post_send_sync()
@staticmethod
def _do_send(
args: IPCTrainerSendWeightsArgs,
names: list[str],
dtype_names: list[str],
shapes: list[list[int]],
ipc_handles: list[dict[str, tuple]] | dict[str, tuple],
tensor_sizes: list[int] | None = None,
packed: bool = False,
) -> None:
"""Send a single update payload via the configured transport."""
update_fields: dict[str, Any] = {
"names": names,
"dtype_names": dtype_names,
"shapes": shapes,
"packed": packed,
}
if tensor_sizes is not None:
update_fields["tensor_sizes"] = tensor_sizes
update_fields["ipc_handles"] = ipc_handles
update_info = IPCWeightTransferUpdateInfo(**update_fields)
if callable(args.send_mode):
args.send_mode(update_info)
elif args.send_mode == "ray":
handles = (
args.llm_handle
if isinstance(args.llm_handle, list)
else [args.llm_handle]
)
ray.get(
args.llm_handle.update_weights.remote(dict(update_info=update_info))
[
h.update_weights.remote(dict(update_info=asdict(update_info)))
for h in handles
]
)
elif args.mode == "http":
# HTTP mode: send via HTTP POST with pickled handles
# Pickle and base64 encode IPC handles for HTTP transmission
elif args.send_mode == "http":
pickled_handles = base64.b64encode(pickle.dumps(ipc_handles)).decode(
"utf-8"
)
http_fields = {k: v for k, v in update_fields.items() if k != "ipc_handles"}
http_fields["ipc_handles_pickled"] = pickled_handles
url = f"{args.url}/update_weights"
payload = {
"update_info": {
"names": names,
"dtype_names": dtype_names,
"shapes": shapes,
"ipc_handles_pickled": pickled_handles,
}
}
payload = {"update_info": http_fields}
response = requests.post(url, json=payload, timeout=300)
response.raise_for_status()
@@ -21,7 +21,7 @@ from vllm.distributed.weight_transfer.base import (
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
DEFAULT_PACKED_NUM_BUFFERS,
packed_broadcast_consumer,
packed_nccl_broadcast_consumer,
)
@@ -184,7 +184,7 @@ class NCCLWeightTransferEngine(
dtype = getattr(torch, dtype_name)
yield (name, (shape, dtype))
packed_broadcast_consumer(
packed_nccl_broadcast_consumer(
iterator=state_dict_info_iterator(),
group=self.model_update_group,
src=0,
@@ -247,10 +247,10 @@ class NCCLWeightTransferEngine(
if args.packed:
# Use packed tensor broadcasting for efficiency
from vllm.distributed.weight_transfer.packed_tensor import (
packed_broadcast_producer,
packed_nccl_broadcast_producer,
)
packed_broadcast_producer(
packed_nccl_broadcast_producer(
iterator=iterator,
group=args.group,
src=args.src,
+283 -72
View File
@@ -4,9 +4,11 @@
import math
from collections.abc import Callable, Iterator
from dataclasses import dataclass
from typing import Any
import torch
from torch.multiprocessing.reductions import reduce_tensor
# Default values for packed tensor configuration.
# These are imported by NCCLWeightTransferUpdateInfo and trainer_send_weights.
@@ -14,7 +16,124 @@ DEFAULT_PACKED_BUFFER_SIZE_BYTES = 1024 * 1024 * 1024 # 1GB
DEFAULT_PACKED_NUM_BUFFERS = 2
def packed_broadcast_producer(
def unpack_tensor(
packed_tensor: torch.Tensor,
names: list[str],
shapes: list[list[int]],
dtypes: list[torch.dtype],
tensor_sizes: list[int],
) -> list[tuple[str, torch.Tensor]]:
"""Unpack a packed uint8 tensor into a list of named tensors.
The returned tensors are **views** of ``packed_tensor`` (the
``.contiguous()`` call is a no-op on already-contiguous row-slices).
If ``packed_tensor`` lives in storage that may be reused — e.g. a
reused CUDA IPC buffer — callers must clone the results before the
underlying storage is overwritten.
Args:
packed_tensor: The packed torch.uint8 tensor to unpack
names: List of tensor names
shapes: List of tensor shapes
dtypes: List of tensor dtypes
tensor_sizes: List of tensor sizes in bytes
"""
unpacked_tensors = packed_tensor.split(tensor_sizes)
return [
(name, tensor.contiguous().view(dtype).view(*shape))
for name, shape, dtype, tensor in zip(names, shapes, dtypes, unpacked_tensors)
]
@dataclass
class PackedChunk:
"""Result of packing tensors into a single contiguous uint8 buffer."""
packed_tensor: torch.Tensor
names: list[str]
shapes: list[list[int]]
dtypes: list[torch.dtype]
tensor_sizes: list[int]
def pack_tensors(
iterator: Iterator[tuple[str, torch.Tensor]],
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
buffer_size_bytes: int,
tensor_list: list[torch.Tensor] | None = None,
current_size: int = 0,
) -> PackedChunk | None:
"""Pack tensors from an iterator into a single contiguous uint8 buffer.
Consumes from the iterator until the accumulated size exceeds
buffer_size_bytes or the iterator is exhausted, then returns a
PackedChunk. Returns None if no tensors were consumed.
Args:
iterator: Iterator of (name, tensor) pairs
post_iter_func: Applied to each item before linearizing to uint8
buffer_size_bytes: Max bytes before flushing
tensor_list: Pre-existing tensor list to append to (for NCCL
multi-buffer reuse). If None, a fresh list is created.
current_size: Byte count already accumulated in tensor_list
"""
if tensor_list is None:
tensor_list = []
names: list[str] = []
shapes: list[list[int]] = []
dtypes: list[torch.dtype] = []
tensor_sizes: list[int] = []
total_bytes = current_size
while True:
try:
item = next(iterator)
except StopIteration:
break
name, orig_tensor = item
# Apply post processing and convert to linearized uint8 tensor
tensor = post_iter_func(item).contiguous().view(torch.uint8).view(-1)
if tensor.numel() > buffer_size_bytes:
import warnings
warnings.warn(
f"Tensor '{name}' has size {tensor.numel()} bytes, which "
f"exceeds buffer_size_bytes={buffer_size_bytes}.",
stacklevel=2,
)
tensor_list.append(tensor)
names.append(name)
shapes.append(list(orig_tensor.shape))
dtypes.append(orig_tensor.dtype)
tensor_sizes.append(tensor.numel())
total_bytes += tensor.numel()
if total_bytes > buffer_size_bytes:
break
if not tensor_list:
return None
packed = torch.cat(tensor_list, dim=0)
del tensor_list
return PackedChunk(
packed_tensor=packed,
names=names,
shapes=shapes,
dtypes=dtypes,
tensor_sizes=tensor_sizes,
)
# ── NCCL packed broadcast ──────────────────────────────────────────────
def packed_nccl_broadcast_producer(
iterator: Iterator[tuple[str, torch.Tensor]],
group: Any,
src: int,
@@ -36,57 +155,31 @@ def packed_broadcast_producer(
Both producer and consumer must use the same value.
"""
target_packed_tensor_size = buffer_size_bytes
streams = [torch.cuda.Stream() for _ in range(num_buffers)]
# Keep references to in-flight chunks so their packed_tensors
# aren't freed while an async broadcast is still reading them.
in_flight: list[PackedChunk | None] = [None] * num_buffers
buffer_idx = 0
packing_tensor_list: list[list[torch.Tensor]] = [[] for _ in range(num_buffers)]
packing_tensor_sizes: list[int] = [0 for _ in range(num_buffers)]
packed_tensors: list[torch.Tensor] = [
torch.empty(0, dtype=torch.uint8, device="cuda") for _ in range(num_buffers)
]
while True:
# Synchronize the current stream
streams[buffer_idx].synchronize()
# Previous chunk on this buffer slot is now safe to free
in_flight[buffer_idx] = None
# Start tasks for the new buffer in a new stream
with torch.cuda.stream(streams[buffer_idx]):
try:
# Initialize the packing tensor list and sizes
packing_tensor_list[buffer_idx] = []
packing_tensor_sizes[buffer_idx] = 0
# Pack the tensors
while True:
# Apply post processing and convert to linearized uint8 tensor
tensor = (
post_iter_func(next(iterator))
.contiguous()
.view(torch.uint8)
.view(-1)
)
packing_tensor_list[buffer_idx].append(tensor)
packing_tensor_sizes[buffer_idx] += tensor.numel()
if packing_tensor_sizes[buffer_idx] > target_packed_tensor_size:
break
# Pack the tensors and call broadcast collective
packed_tensors[buffer_idx] = torch.cat(
packing_tensor_list[buffer_idx], dim=0
)
group.broadcast(packed_tensors[buffer_idx], src=src)
# Move to the next buffer
buffer_idx = (buffer_idx + 1) % num_buffers
except StopIteration:
# Do the last broadcast if there are remaining tensors
if len(packing_tensor_list[buffer_idx]) > 0:
packed_tensors[buffer_idx] = torch.cat(
packing_tensor_list[buffer_idx], dim=0
)
group.broadcast(packed_tensors[buffer_idx], src=src)
chunk = pack_tensors(iterator, post_iter_func, buffer_size_bytes)
if chunk is None:
break
# Pack the tensors and call broadcast collective
group.broadcast(chunk.packed_tensor, src=src)
# Hold reference until this stream is synchronized
in_flight[buffer_idx] = chunk
# Move to the next buffer
buffer_idx = (buffer_idx + 1) % num_buffers
def packed_broadcast_consumer(
def packed_nccl_broadcast_consumer(
iterator: Iterator[tuple[str, tuple[list[int], torch.dtype]]],
group: Any,
src: int,
@@ -108,37 +201,6 @@ def packed_broadcast_consumer(
Both producer and consumer must use the same value.
"""
def unpack_tensor(
packed_tensor: torch.Tensor,
names: list[str],
shapes: list[list[int]],
dtypes: list[torch.dtype],
tensor_sizes: list[int],
) -> list[tuple[str, torch.Tensor]]:
"""Unpack a single tensor into a list of tensors.
Args:
packed_tensor: The packed torch.uint8 tensor to unpack
names: List of tensor names
shapes: List of tensor shapes
dtypes: List of tensor dtypes
tensor_sizes: List of tensor sizes in bytes
Returns:
unpacked List[(name, tensor)]
"""
unpacked_tensors = packed_tensor.split(tensor_sizes)
unpacked_list = [
(name, tensor.contiguous().view(dtype).view(*shape))
for name, shape, dtype, tensor in zip(
names, shapes, dtypes, unpacked_tensors
)
]
return unpacked_list
target_packed_tensor_size = buffer_size_bytes
streams = [torch.cuda.Stream() for _ in range(num_buffers)]
@@ -214,3 +276,152 @@ def packed_broadcast_consumer(
)
)
break
# ── IPC packed transfer ────────────────────────────────────────────────
@dataclass
class PackedIpcChunk:
"""Metadata and IPC handle for a single packed chunk."""
names: list[str]
shapes: list[list[int]]
dtype_names: list[str]
tensor_sizes: list[int]
ipc_handle: dict[str, tuple]
def packed_ipc_producer(
iterator: Iterator[tuple[str, torch.Tensor]],
gpu_uuid: str,
post_iter_func: Callable[[tuple[str, torch.Tensor]], torch.Tensor],
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
) -> Iterator[PackedIpcChunk]:
"""Pack tensors into a reusable IPC buffer and yield handles.
Allocates a single GPU buffer of ``buffer_size_bytes`` and registers
it for IPC once via ``reduce_tensor``. Each chunk's packed data is
copied into this buffer before yielding, so only one IPC-shared
allocation exists for the lifetime of the transfer.
Callers **must** ensure the consumer has finished reading the buffer
(e.g. ``ray.get`` returned) before resuming the generator for the
next chunk.
Args:
iterator: Iterator of (name, tensor) pairs.
gpu_uuid: Physical GPU UUID string for this rank.
post_iter_func: Applied to each (name, tensor) before packing.
buffer_size_bytes: Exact capacity of the reusable IPC buffer.
Every chunk is guaranteed to fit within this size. A
``ValueError`` is raised if any single tensor exceeds it.
"""
ipc_buffer = torch.empty(buffer_size_bytes, dtype=torch.uint8, device="cuda")
_, ipc_args = reduce_tensor(ipc_buffer)
names: list[str] = []
shapes: list[list[int]] = []
dtypes: list[torch.dtype] = []
tensor_sizes: list[int] = []
total_bytes = 0
for name, orig_tensor in iterator:
flat = (
post_iter_func((name, orig_tensor)).contiguous().view(torch.uint8).view(-1)
)
if flat.numel() > buffer_size_bytes:
raise ValueError(
f"Tensor '{name}' has size {flat.numel()} bytes, "
f"which exceeds buffer_size_bytes={buffer_size_bytes}. "
f"Increase buffer_size_bytes to at least {flat.numel()}."
)
if total_bytes and total_bytes + flat.numel() > buffer_size_bytes:
# Drain queued copies so the consumer sees a fully-written buffer.
torch.cuda.current_stream().synchronize()
yield PackedIpcChunk(
names=names,
shapes=shapes,
dtype_names=[str(d).split(".")[-1] for d in dtypes],
tensor_sizes=tensor_sizes,
ipc_handle={gpu_uuid: ipc_args},
)
# Rebind to fresh lists so the yielded chunk's metadata is
# not mutated while the consumer is still reading.
names, shapes, dtypes, tensor_sizes = [], [], [], []
total_bytes = 0
ipc_buffer[total_bytes : total_bytes + flat.numel()].copy_(flat)
names.append(name)
shapes.append(list(orig_tensor.shape))
dtypes.append(orig_tensor.dtype)
tensor_sizes.append(flat.numel())
total_bytes += flat.numel()
if total_bytes:
torch.cuda.current_stream().synchronize()
yield PackedIpcChunk(
names=names,
shapes=shapes,
dtype_names=[str(d).split(".")[-1] for d in dtypes],
tensor_sizes=tensor_sizes,
ipc_handle={gpu_uuid: ipc_args},
)
def packed_ipc_consumer(
ipc_handle: dict[str, tuple],
names: list[str],
shapes: list[list[int]],
dtype_names: list[str],
tensor_sizes: list[int],
device_index: int,
) -> list[tuple[str, torch.Tensor]]:
"""Unpack a single packed IPC chunk into named tensors.
Reconstructs the packed buffer via rebuild_cuda_tensor, unpacks
into individual tensors, and clones each into independent storage
before returning.
The clone is intentional: the producer reuses one IPC buffer across
chunks, so any tensor view that aliases the buffer would observe the
*next* chunk's bytes as soon as the producer's generator is resumed.
Callers that retain references past their own update_weights call
(notably vLLM's layerwise reload, which buffers ``bound_args`` for
replay in ``_layerwise_process``) would otherwise replay against
stale data and silently corrupt multi-chunk weight transfers.
Args:
ipc_handle: Mapping of GPU UUID to rebuild_cuda_tensor args tuple
names: Parameter names in the packed buffer
shapes: Parameter shapes
dtype_names: Parameter dtype name strings (e.g. "float16")
tensor_sizes: Size in bytes of each parameter in the packed buffer
device_index: Local CUDA device index
"""
from torch.multiprocessing.reductions import rebuild_cuda_tensor
props = torch.cuda.get_device_properties(device_index)
physical_gpu_id = str(props.uuid)
if physical_gpu_id not in ipc_handle:
raise ValueError(
f"IPC handle not found for GPU UUID {physical_gpu_id}. "
f"Available UUIDs: {list(ipc_handle.keys())}"
)
args = ipc_handle[physical_gpu_id]
list_args = list(args)
list_args[6] = device_index
packed = rebuild_cuda_tensor(*list_args)
content_size = sum(tensor_sizes)
packed = packed[:content_size]
dtypes = [getattr(torch, dn) for dn in dtype_names]
return [
(name, t.clone())
for name, t in unpack_tensor(packed, names, shapes, dtypes, tensor_sizes)
]