[Perf] Zero-copy torch.Tensor pickling in shm_broadcast MessageQueue (#48442)

Signed-off-by: Ruinan Ma <r7ma3088@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Ruinan Ma
2026-07-28 16:28:28 -07:00
committed by GitHub
co-authored by Nick Hill
parent bb3b61f2fd
commit a07fac758f
3 changed files with 233 additions and 3 deletions
+149
View File
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import pickle
import random import random
import threading import threading
import time import time
@@ -10,12 +12,15 @@ from unittest import mock
import multiprocess as mp import multiprocess as mp
import numpy as np import numpy as np
import pytest import pytest
import torch
import torch.distributed as dist import torch.distributed as dist
from vllm.distributed.device_communicators import shm_broadcast from vllm.distributed.device_communicators import shm_broadcast
from vllm.distributed.device_communicators.shm_broadcast import ( from vllm.distributed.device_communicators.shm_broadcast import (
MessageQueue, MessageQueue,
ShmRingBuffer, ShmRingBuffer,
_rebuild_tensor,
_reduce_tensor,
check_shm_free_space, check_shm_free_space,
) )
from vllm.distributed.utils import StatelessProcessGroup from vllm.distributed.utils import StatelessProcessGroup
@@ -354,6 +359,150 @@ def test_message_queue_busy_to_idle():
distributed_run(worker_fn_test_busy_to_idle, 4) distributed_run(worker_fn_test_busy_to_idle, 4)
@worker_fn_wrapper
def worker_fn_tensor_broadcast():
rank = dist.get_rank()
writer_rank = 0
message_queue = MessageQueue.create_from_process_group(
dist.group.WORLD, 8 * 1024 * 1024, 4, writer_rank
)
# Both ranks construct the identical reference payload.
torch.manual_seed(42)
payload = {
# 2MiB: rides the shm ring as an out-of-band buffer (the receiving
# side must copy out of the reusable ring chunk).
"mid": torch.randn(1024, 512),
# 16MiB > max_chunk_bytes: overflows to the zmq socket (the
# receiving side aliases the zmq.Frame zero-copy).
"big": torch.randn(4096, 2048, dtype=torch.bfloat16),
"nested": ["plain", 123, {"inner": torch.arange(5)}],
}
if rank == writer_rank:
with mock.patch(
"vllm.distributed.device_communicators.shm_broadcast._reduce_tensor",
wraps=_reduce_tensor,
) as wrapped_reduce:
message_queue.enqueue(payload)
assert wrapped_reduce.call_count == 3
# Cycle the ring (max_chunks=4) several times over so that aliased
# ring chunks would be overwritten.
for i in range(16):
message_queue.enqueue({"junk": torch.full((1024, 512), float(i))})
else:
received = message_queue.dequeue(timeout=30)
for key in ("mid", "big"):
assert torch.equal(received[key], payload[key]), key
assert received[key].dtype == payload[key].dtype, key
assert torch.equal(received["nested"][2]["inner"], torch.arange(5))
snapshot = received["mid"].clone()
for i in range(16):
junk = message_queue.dequeue(timeout=30)
assert torch.equal(junk["junk"], torch.full((1024, 512), float(i)))
# Tensors received via the shm ring must not alias chunk memory
# that the writer has reused for subsequent messages.
assert torch.equal(received["mid"], snapshot)
# Rebuilt tensors must be writable, like regular tensors.
received["mid"] += 1.0
received["big"][0, 0] = 1.0
dist.barrier()
print(f"tensor broadcast passed the test! Rank {rank}")
def test_tensor_broadcast():
distributed_run(worker_fn_tensor_broadcast, 2)
def _dumps_oob(obj) -> tuple[bytes, list]:
"""Pickle `obj` the same way `MessageQueue.enqueue` does: tensor
dispatch table + out-of-band buffers >= 1MiB."""
buffers = []
def callback(buf: pickle.PickleBuffer) -> bool:
raw = buf.raw()
if raw.nbytes < 1024 * 1024:
return True
buffers.append(raw)
return False
bio = io.BytesIO()
pickler = pickle.Pickler(
bio, protocol=pickle.HIGHEST_PROTOCOL, buffer_callback=callback
)
pickler.dispatch_table = {torch.Tensor: _reduce_tensor}
pickler.dump(obj)
return bio.getvalue(), buffers
@pytest.mark.parametrize(
"case",
[
"small",
"mid",
"bf16",
"fp8",
"empty",
"scalar",
"noncontig",
"requires_grad",
"conj",
"param",
],
)
def test_tensor_pickle_roundtrip(case: str):
tensor = {
# Inlined in-band (< 1MiB) and out-of-band (>= 1MiB) buffers.
"small": lambda: torch.randn(100, 10),
"mid": lambda: torch.randn(1024, 512),
# Dtypes numpy doesn't recognize.
"bf16": lambda: torch.randn(512, 512, dtype=torch.bfloat16),
"fp8": lambda: torch.randn(32, 32).to(torch.float8_e4m3fn),
# Shape edge cases.
"empty": lambda: torch.empty(0, 8),
"scalar": lambda: torch.tensor(3.14),
"noncontig": lambda: torch.randn(64, 64).t(),
# These fall back to torch's default reducer.
"requires_grad": lambda: torch.randn(8, 8, requires_grad=True),
"conj": lambda: torch.randn(4, dtype=torch.complex64).conj(),
"param": lambda: torch.nn.Parameter(torch.randn(4), requires_grad=False),
}[case]()
data, buffers = _dumps_oob({"tensor": tensor, "meta": list(range(10))})
received = pickle.loads(data, buffers=buffers)["tensor"]
assert received.shape == tensor.shape
assert received.dtype == tensor.dtype
if tensor.dtype == torch.float8_e4m3fn:
assert torch.equal(received.view(torch.uint8), tensor.view(torch.uint8))
else:
assert torch.equal(received, tensor)
assert received.requires_grad == tensor.requires_grad
assert isinstance(received, type(tensor))
if tensor.numel() and not tensor.requires_grad:
# Rebuilt tensors must be writable, like regular tensors.
received.view(-1)[0] = 1.0
@pytest.mark.parametrize("case", ["cuda", "requires_grad", "conj"])
def test_reduce_tensor_fallback(case: str):
"""Tensors the zero-copy reducer can't safely alias must fall back to
torch's default reduction."""
if case == "cuda":
if not torch.cuda.is_available():
pytest.skip("requires CUDA")
tensor = torch.randn(4, device="cuda")
elif case == "requires_grad":
tensor = torch.randn(8, requires_grad=True)
else:
tensor = torch.randn(4, dtype=torch.complex64).conj()
reduced = _reduce_tensor(tensor)
assert reduced[0] is not _rebuild_tensor
@pytest.mark.parametrize("should_warn", [False, True]) @pytest.mark.parametrize("should_warn", [False, True])
def test_reader_timeout_caps_indefinite_waits(should_warn): def test_reader_timeout_caps_indefinite_waits(should_warn):
with ( with (
@@ -48,6 +48,7 @@ CHECK_IMPORTS = {
"vllm/distributed/device_communicators/shm_object_storage.py", "vllm/distributed/device_communicators/shm_object_storage.py",
"vllm/distributed/weight_transfer/ipc_engine.py", "vllm/distributed/weight_transfer/ipc_engine.py",
"vllm/distributed/weight_transfer/clients.py", "vllm/distributed/weight_transfer/clients.py",
"tests/distributed/test_shm_broadcast.py",
"tests/distributed/test_weight_transfer.py", "tests/distributed/test_weight_transfer.py",
"vllm/utils/hashing.py", "vllm/utils/hashing.py",
"tests/multimodal/media/test_base.py", "tests/multimodal/media/test_base.py",
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copyreg
import functools import functools
import io
import os import os
import pickle import pickle
import shutil import shutil
@@ -385,6 +387,70 @@ class ShmRingBuffer:
yield buf yield buf
def _rebuild_tensor(buf: Any, shape: tuple[int, ...], dtype_str: str) -> torch.Tensor:
"""Rebuild a tensor from an out-of-band pickle buffer.
Counterpart of `_reduce_tensor`. Note that pickle passes the original
buffer-providing object from `loads(buffers=...)` straight to this
function (no `PickleBuffer` wrapper on the receiving side), so `buf` is
a `zmq.Frame`, a `memoryview` of a shared-memory ring chunk, or `bytes`
if the buffer was serialized in-band.
"""
dtype = getattr(torch, dtype_str)
assert isinstance(dtype, torch.dtype)
if isinstance(buf, zmq.Frame):
# ZMQ frames own their message memory independently of any context,
# so the tensor can safely alias it with zero copies. The tensor's
# storage keeps the frame (and thus its bytes) alive via a strong
# reference for as long as the tensor is.
try:
return torch.frombuffer(buf, dtype=torch.uint8).view(dtype).view(shape)
except ValueError:
# Empty or read-only frame buffer; fall through to the copy path.
pass
# Shared-memory ring buffer chunks are reused by the writer once all
# readers have marked them read, so we must copy out of them. bytearray
# (vs bytes) keeps the resulting tensor writable, matching normal tensor
# semantics.
raw = bytearray(buf)
if not raw:
assert 0 in shape
return torch.empty(shape, dtype=dtype)
return torch.frombuffer(raw, dtype=torch.uint8).view(dtype).view(shape)
def _reduce_tensor(tensor: torch.Tensor):
"""Reduce a CPU tensor to a `PickleBuffer` for out-of-band pickling.
`torch.Tensor.__reduce_ex__` copies the tensor bytes into the pickle
byte stream via `torch.serialization` and never emits a `PickleBuffer`,
which defeats the out-of-band buffer handling in `MessageQueue.enqueue`.
This reducer instead exposes the tensor's memory directly, so large
tensors (e.g. `prompt_embeds` in `SchedulerOutput`) traverse the queue
without being copied into and back out of the pickled message.
"""
if (
tensor.device.type == "cpu"
and tensor.layout == torch.strided
and not tensor.requires_grad
):
try:
# The uint8 view exposes the raw bytes via the buffer protocol,
# including for dtypes numpy doesn't recognize (bfloat16, fp8, ...).
# reshape(-1) first so that 0-dim tensors can be viewed as well.
raw = tensor.contiguous().reshape(-1).view(torch.uint8).numpy()
except RuntimeError:
# Exotic tensors (e.g. with the conjugate bit set) that don't
# support aliasing views; let torch handle them.
pass
else:
dtype_str = str(tensor.dtype).removeprefix("torch.")
return _rebuild_tensor, (PickleBuffer(raw), tuple(tensor.shape), dtype_str)
# Fall back to torch's default (copying) reduction.
return tensor.__reduce_ex__(pickle.HIGHEST_PROTOCOL)
@dataclass @dataclass
class Handle: class Handle:
local_reader_ranks: list[int] = field(default_factory=list) local_reader_ranks: list[int] = field(default_factory=list)
@@ -771,9 +837,23 @@ class MessageQueue:
total_bytes += len(raw_buf) + 4 total_bytes += len(raw_buf) + 4
return False return False
all_buffers[0] = pickle.dumps( # CPU tensors are routed through `_reduce_tensor` so that their
obj, protocol=pickle.HIGHEST_PROTOCOL, buffer_callback=oob_callback # bytes are emitted as out-of-band buffers instead of being
) # copied into the pickle stream by torch's default reducer.
# Start from `copyreg.dispatch_table` to preserve globally
# registered reducers (e.g. `re.Pattern`); the per-pickler
# dispatch table would otherwise shadow them.
dispatch_table = dict(copyreg.dispatch_table)
dispatch_table[torch.Tensor] = _reduce_tensor
with io.BytesIO() as bio:
pickler = pickle.Pickler(
bio,
protocol=pickle.HIGHEST_PROTOCOL,
buffer_callback=oob_callback,
)
pickler.dispatch_table = dispatch_table
pickler.dump(obj)
all_buffers[0] = bio.getvalue()
if self.n_local_reader > 0: if self.n_local_reader > 0:
if total_bytes + len(all_buffers[0]) >= self.buffer.max_chunk_bytes: if total_bytes + len(all_buffers[0]) >= self.buffer.max_chunk_bytes:
with self.acquire_write(timeout) as buf: with self.acquire_write(timeout) as buf: