[Perf] Zero-copy torch.Tensor pickling in shm_broadcast MessageQueue (#48442)

Signed-off-by: Ruinan Ma <r7ma3088@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Ruinan Ma
2026-07-28 16:28:28 -07:00
committed by GitHub
co-authored by Nick Hill
parent bb3b61f2fd
commit a07fac758f
3 changed files with 233 additions and 3 deletions
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copyreg
import functools
import io
import os
import pickle
import shutil
@@ -385,6 +387,70 @@ class ShmRingBuffer:
yield buf
def _rebuild_tensor(buf: Any, shape: tuple[int, ...], dtype_str: str) -> torch.Tensor:
"""Rebuild a tensor from an out-of-band pickle buffer.
Counterpart of `_reduce_tensor`. Note that pickle passes the original
buffer-providing object from `loads(buffers=...)` straight to this
function (no `PickleBuffer` wrapper on the receiving side), so `buf` is
a `zmq.Frame`, a `memoryview` of a shared-memory ring chunk, or `bytes`
if the buffer was serialized in-band.
"""
dtype = getattr(torch, dtype_str)
assert isinstance(dtype, torch.dtype)
if isinstance(buf, zmq.Frame):
# ZMQ frames own their message memory independently of any context,
# so the tensor can safely alias it with zero copies. The tensor's
# storage keeps the frame (and thus its bytes) alive via a strong
# reference for as long as the tensor is.
try:
return torch.frombuffer(buf, dtype=torch.uint8).view(dtype).view(shape)
except ValueError:
# Empty or read-only frame buffer; fall through to the copy path.
pass
# Shared-memory ring buffer chunks are reused by the writer once all
# readers have marked them read, so we must copy out of them. bytearray
# (vs bytes) keeps the resulting tensor writable, matching normal tensor
# semantics.
raw = bytearray(buf)
if not raw:
assert 0 in shape
return torch.empty(shape, dtype=dtype)
return torch.frombuffer(raw, dtype=torch.uint8).view(dtype).view(shape)
def _reduce_tensor(tensor: torch.Tensor):
"""Reduce a CPU tensor to a `PickleBuffer` for out-of-band pickling.
`torch.Tensor.__reduce_ex__` copies the tensor bytes into the pickle
byte stream via `torch.serialization` and never emits a `PickleBuffer`,
which defeats the out-of-band buffer handling in `MessageQueue.enqueue`.
This reducer instead exposes the tensor's memory directly, so large
tensors (e.g. `prompt_embeds` in `SchedulerOutput`) traverse the queue
without being copied into and back out of the pickled message.
"""
if (
tensor.device.type == "cpu"
and tensor.layout == torch.strided
and not tensor.requires_grad
):
try:
# The uint8 view exposes the raw bytes via the buffer protocol,
# including for dtypes numpy doesn't recognize (bfloat16, fp8, ...).
# reshape(-1) first so that 0-dim tensors can be viewed as well.
raw = tensor.contiguous().reshape(-1).view(torch.uint8).numpy()
except RuntimeError:
# Exotic tensors (e.g. with the conjugate bit set) that don't
# support aliasing views; let torch handle them.
pass
else:
dtype_str = str(tensor.dtype).removeprefix("torch.")
return _rebuild_tensor, (PickleBuffer(raw), tuple(tensor.shape), dtype_str)
# Fall back to torch's default (copying) reduction.
return tensor.__reduce_ex__(pickle.HIGHEST_PROTOCOL)
@dataclass
class Handle:
local_reader_ranks: list[int] = field(default_factory=list)
@@ -771,9 +837,23 @@ class MessageQueue:
total_bytes += len(raw_buf) + 4
return False
all_buffers[0] = pickle.dumps(
obj, protocol=pickle.HIGHEST_PROTOCOL, buffer_callback=oob_callback
)
# CPU tensors are routed through `_reduce_tensor` so that their
# bytes are emitted as out-of-band buffers instead of being
# copied into the pickle stream by torch's default reducer.
# Start from `copyreg.dispatch_table` to preserve globally
# registered reducers (e.g. `re.Pattern`); the per-pickler
# dispatch table would otherwise shadow them.
dispatch_table = dict(copyreg.dispatch_table)
dispatch_table[torch.Tensor] = _reduce_tensor
with io.BytesIO() as bio:
pickler = pickle.Pickler(
bio,
protocol=pickle.HIGHEST_PROTOCOL,
buffer_callback=oob_callback,
)
pickler.dispatch_table = dispatch_table
pickler.dump(obj)
all_buffers[0] = bio.getvalue()
if self.n_local_reader > 0:
if total_bytes + len(all_buffers[0]) >= self.buffer.max_chunk_bytes:
with self.acquire_write(timeout) as buf: