[Core] shm tensor arena for efficient cpu->gpu worker broadcast (#51207)

Signed-off-by: Bolin Sun <bolins@nvidia.com>
Signed-off-by: <bolins@nvidia.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Bolin Sun
2026-10-09 16:13:24 -07:00
committed by GitHub
co-authored by Claude Nick Hill
parent fa94ad4143
commit 04b02fc8a5
8 changed files with 1290 additions and 8 deletions
+10
View File
@@ -235,6 +235,15 @@ class ParallelConfig:
disable_custom_all_reduce: bool = False
"""Disable the custom all-reduce kernel and fall back to NCCL."""
enable_shm_tensor_arena: bool = False
"""Route large CPU tensors (e.g. multimodal ``pixel_values``) in the
engine→worker broadcast through a zero-copy shared-memory arena instead of
sending a copy to every reader. **Experimental**: may be reconciled with
vLLM's existing multimodal shm tensor caching in a future release. Opt-in;
most beneficial for multimodal serving with large images and tensor
parallelism. Active only when all queue readers are node-local; reserves
slots in ``/dev/shm``."""
enable_elastic_ep: bool = False
"""Enable elastic expert parallelism with stateless NCCL groups for DP/EP."""
elastic_ep_max_dp_size: int = Field(default=None, ge=1) # type: ignore[assignment]
@@ -901,6 +910,7 @@ class ParallelConfig:
"nnodes",
"max_parallel_loading_workers",
"disable_custom_all_reduce",
"enable_shm_tensor_arena",
"ray_workers_use_nsight",
"ray_runtime_env",
"placement_group",
@@ -61,6 +61,10 @@ SPINLOOP_TIMEOUT_SECONDS = 0.1
if TYPE_CHECKING:
from _typeshed import SizedBuffer
from vllm.distributed.device_communicators.shm_tensor_arena import (
ShmTensorArena,
)
VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
# Cap on how long an idle reader parks before re-reading the authoritative SHM
# written-flag. Bounds lost-notify recovery latency to ~5s while the periodic
@@ -471,6 +475,7 @@ class Handle:
local_notify_addr: str | None = None
remote_subscribe_addr: str | None = None
remote_addr_ipv6: bool = False
tensor_arena_handle: tuple[int, int, int, str] | None = None
def _close_zmq(context: zmq.Context, sockets: list[zmq.Socket]) -> None:
@@ -490,6 +495,7 @@ class MessageQueue:
max_chunk_bytes: int = 1024 * 1024 * 24,
max_chunks: int = 10,
connect_ip: str | None = None,
enable_shm_tensor_arena: bool = False,
):
if local_reader_ranks is None:
local_reader_ranks = list(range(n_local_reader))
@@ -501,12 +507,22 @@ class MessageQueue:
self.shutting_down = False
context = Context()
self.tensor_arena: ShmTensorArena | None = None
if n_local_reader > 0:
# for local readers, we will:
# 1. create a shared memory ring buffer to communicate small data
# 2. create a publish-subscribe socket to communicate large data
self.buffer = ShmRingBuffer(n_local_reader, max_chunk_bytes, max_chunks)
# Zero-copy arena for large CPU tensors (see ShmTensorArena).
# Local readers only: remote readers receive the pickled bytes
# over a socket and cannot map the arena, so the substitution
# would break them.
if enable_shm_tensor_arena and n_remote_reader == 0:
from vllm.distributed.device_communicators import shm_tensor_arena
self.tensor_arena = shm_tensor_arena.ShmTensorArena(n_local_reader)
# XPUB is very similar to PUB,
# except that it can receive subscription messages
# to confirm the number of subscribers
@@ -567,6 +583,9 @@ class MessageQueue:
local_notify_addr=local_notify_addr,
remote_subscribe_addr=remote_subscribe_addr,
remote_addr_ipv6=remote_addr_ipv6,
tensor_arena_handle=(
self.tensor_arena.handle() if self.tensor_arena is not None else None
),
)
self._close_zmq_on_collect(context)
@@ -599,6 +618,7 @@ class MessageQueue:
context = Context()
self.tensor_arena = None
if rank in handle.local_reader_ranks:
assert handle.buffer_handle is not None
self.buffer = ShmRingBuffer(*handle.buffer_handle)
@@ -607,6 +627,23 @@ class MessageQueue:
self._is_local_reader = True
self._is_remote_reader = False
arena_handle = handle.tensor_arena_handle
if arena_handle is not None:
from vllm.distributed.device_communicators import shm_tensor_arena
self.tensor_arena = shm_tensor_arena.ShmTensorArena(
*arena_handle, reader_rank=self.local_reader_rank
)
# `shared_memory` is unset (not just None) if attaching to
# the writer's segment raised FileNotFoundError -- see
# ShmTensorArena.__init__'s reader branch. Registering a
# broken arena is a no-op deferred failure (same convention
# ShmRingBuffer uses above), not a crash here.
if hasattr(self.tensor_arena, "shared_memory"):
shm_tensor_arena._TENSOR_ARENAS[
self.tensor_arena.shared_memory.name
] = self.tensor_arena
self.local_socket = context.socket(SUB)
self.local_socket.setsockopt_string(SUBSCRIBE, "")
socket_addr = handle.local_subscribe_addr
@@ -880,12 +917,21 @@ class MessageQueue:
# dispatch table would otherwise shadow them.
dispatch_table = dict(copyreg.dispatch_table)
dispatch_table[torch.Tensor] = _reduce_tensor
arena = self.tensor_arena
with io.BytesIO() as bio:
pickler = pickle.Pickler(
bio,
protocol=pickle.HIGHEST_PROTOCOL,
buffer_callback=oob_callback,
)
pickler: pickle.Pickler
if arena is not None:
from vllm.distributed.device_communicators.shm_tensor_arena import (
_ArenaPickler,
)
pickler = _ArenaPickler(bio, arena, buffer_callback=oob_callback)
else:
pickler = pickle.Pickler(
bio,
protocol=pickle.HIGHEST_PROTOCOL,
buffer_callback=oob_callback,
)
pickler.dispatch_table = dispatch_table
pickler.dump(obj)
all_buffers[0] = bio.getvalue()
@@ -922,6 +968,9 @@ class MessageQueue:
):
"""Read from message queue with optional timeout (in seconds)"""
if self._is_local_reader:
# See ShmTensorArena.flush_releases for what this retires and why.
if self.tensor_arena is not None:
self.tensor_arena.flush_releases()
with self.acquire_read(timeout, indefinite) as buf:
overflow = buf[0] == 1
if not overflow:
@@ -0,0 +1,397 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Zero-copy shared-memory arena for large CPU tensors (multimodal fast path).
Problem: `MessageQueue.enqueue` (see `shm_broadcast.py`) pickles the whole
payload, and torch's default tensor pickling copies bytes into the pickle
stream on the writer and back out on every local reader. For a large
`pixel_values` tensor (up to hundreds of MB) this blocks the EngineCore step
loop for ~1s with all GPUs idle.
Fast path: `_ArenaPickler.reducer_override` intercepts qualifying CPU
tensors during pickling, memcpys them ONCE into a free arena slot, and
pickles a tiny `(arena_name, slot, nbytes, dtype, shape)` stub. Readers
rebuild a zero-copy view of the mapped slot (`torch.frombuffer`).
Slot lifecycle: single writer, n_reader readers, per-slot metadata
[written_flag, reader0_done, ..., readerN_done] (same protocol as
ShmRingBuffer). A slot is not released when rebuilt: the rebuilt tensor is
the SOURCE of an async H2D copy, and callers like chunked-prefill
`prompt_embeds` retain it across many steps. Release is queued only once the
tensor's storage is freed, i.e. once no tensor or view of the slot remains
(see `get_tensor`), and is further gated on a CUDA event on the pinned fast
path, since the H2D can outlive the step that issued it (see
`flush_releases`).
The writer NEVER blocks: no free slot (or an oversized tensor) falls back to
the default pickle path, so worst case matches today's behavior.
Physical memory: slots are paged in lazily, so an arena on a queue that
never carries big tensors costs ~0. Creation is guarded by
`check_shm_free_space` (like `ShmRingBuffer`) so an undersized `/dev/shm`
fails fast instead of a `SIGBUS` mid-copy.
"""
import pickle
import weakref
from contextlib import suppress
from multiprocessing import shared_memory
from unittest.mock import patch
import numpy as np
import torch
from vllm.distributed.device_communicators.shm_broadcast import (
check_shm_free_space,
memory_fence,
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
logger = init_logger(__name__)
# Arena sizing — internal constants (kept off the env/CLI surface).
_ARENA_SLOTS = 8
_ARENA_SLOT_BYTES = 256 * 1024 * 1024
# Minimum contiguous CPU-tensor size to divert into the arena; smaller tensors
# take the out-of-band pickle-buffer path.
_ARENA_MIN_BYTES = 8 * 1024 * 1024
# Reader-side registry: arena shm name -> attached ShmTensorArena, keyed by
# name rather than a single slot since a process isn't guaranteed to attach
# to only one. Populated by `MessageQueue.create_from_handle`; consumed by
# the `_rebuild_arena_*` unpickle hooks. WEAK-value: a strong reference here
# would keep the arena (and its pinned mapping) alive for the process
# lifetime, and would need `MessageQueue.shutdown` to remove it explicitly --
# unsafe since shutdown() can run on a different thread than an in-flight
# `dequeue`. Instead an entry just disappears once nothing else references
# that arena.
_TENSOR_ARENAS: weakref.WeakValueDictionary[str, "ShmTensorArena"] = (
weakref.WeakValueDictionary()
)
class ShmTensorArena:
"""Slotted shared-memory arena: one writer, n_reader zero-copy readers.
Memory layout: [slot0 | slot1 | ... | slotN-1 | meta0 | meta1 | ... ]
where each meta is (1 + n_reader) bytes: [written_flag, reader_done...].
Slot states mirror ShmRingBuffer:
written=0 -> free (never written / being written)
written=1, some reader_done=0 -> in use, cannot reuse
written=1, all reader_done=1 -> consumed, can reuse
"""
def __init__(
self,
n_reader: int,
slot_bytes: int = _ARENA_SLOT_BYTES,
n_slots: int = _ARENA_SLOTS,
name: str | None = None,
reader_rank: int = -1,
):
self.n_reader = n_reader
self.slot_bytes = slot_bytes
self.n_slots = n_slots
self.reader_rank = reader_rank # -1 for the writer
self.metadata_size = 1 + n_reader
self.metadata_offset = slot_bytes * n_slots
self.total_bytes = (slot_bytes + self.metadata_size) * n_slots
self._next_slot = 0
self._fallbacks = 0
self._pin_attempted = False
self._pinned = False
self._pinned_ptr = 0
# slots queued for release once the storage of the tensor `get_tensor`
# handed out is freed; flushed at the next dequeue.
self._pending_release: list[int] = []
# slots whose release is gated on an H2D-completion CUDA event: each
# entry is (event, [slot_idx, ...]). Only populated on the pinned fast
# path, where the source-side DMA is asynchronous (see flush_releases).
self._deferred_releases: list[tuple[torch.cuda.Event, list[int]]] = []
if name is None:
self.is_creator = True
# Same guard as ShmRingBuffer: SharedMemory/ftruncate can succeed
# on an undersized /dev/shm, so check up front instead of SIGBUS
# on first write past tmpfs capacity.
check_shm_free_space(self.total_bytes, allocation_name="ShmTensorArena")
self.shared_memory = shared_memory.SharedMemory(
create=True, size=self.total_bytes
)
assert self.shared_memory.buf is not None
with self.shared_memory.buf[self.metadata_offset :] as meta:
torch.frombuffer(meta, dtype=torch.uint8).fill_(0)
else:
self.is_creator = False
# same resource_tracker workaround as ShmRingBuffer
with patch(
"multiprocessing.resource_tracker.register",
lambda *args, **kwargs: None,
):
try:
self.shared_memory = shared_memory.SharedMemory(name=name)
assert self.shared_memory.size >= self.total_bytes
except FileNotFoundError:
# deserialized on a different node; arena unused there
pass
def handle(self):
return (self.n_reader, self.slot_bytes, self.n_slots, self.shared_memory.name)
def _meta(self, idx: int) -> memoryview:
start = self.metadata_offset + idx * self.metadata_size
assert self.shared_memory.buf is not None
return self.shared_memory.buf[start : start + self.metadata_size]
def _slot(self, idx: int, nbytes: int) -> memoryview:
start = idx * self.slot_bytes
assert self.shared_memory.buf is not None
return self.shared_memory.buf[start : start + nbytes]
# ---- writer side ----
def write_tensor(self, t: torch.Tensor) -> int | None:
"""Copy tensor bytes into a free slot; return slot idx or None
(caller must then fall back to the default pickle path)."""
nbytes = t.numel() * t.element_size()
if nbytes > self.slot_bytes:
return None
try:
# Computed before claiming a slot: exotic tensors (e.g. conjugate
# bit set) raise RuntimeError here, same as `_reduce_tensor`'s
# identical view -- fail before claiming, not after.
src = t.detach().reshape(-1).view(torch.uint8)
except RuntimeError:
return None
memory_fence()
for probe in range(self.n_slots):
idx = (self._next_slot + probe) % self.n_slots
with self._meta(idx) as meta:
free = meta[0] == 0 or sum(meta[1:]) == self.n_reader
if not free:
continue
meta[0] = 0 # claim
slot_mv = self._slot(idx, nbytes)
try:
dst = torch.frombuffer(slot_mv, dtype=torch.uint8, count=nbytes)
dst.copy_(src)
finally:
slot_mv.release()
with self._meta(idx) as meta:
for i in range(1, self.n_reader + 1):
meta[i] = 0
memory_fence()
meta[0] = 1
memory_fence()
self._next_slot = (idx + 1) % self.n_slots
return idx
self._fallbacks += 1
if self._fallbacks == 1 or self._fallbacks % 100 == 0:
logger.info(
"ShmTensorArena: no free slot (%d bytes, %d fallbacks so far); "
"falling back to the out-of-band pickle path.",
nbytes,
self._fallbacks,
)
return None
# ---- reader side ----
def _ensure_pinned(self):
"""Pin the whole arena mapping via cudaHostRegister in THIS process
(lazy, once). Without it the HtoD of a zero-copy tensor pays
first-touch page faults on the tmpfs mapping plus pageable staging
(~hundreds of ms for 200MB); registration allocates+pins the pages
once, making every later HtoD a true DMA. Failure (no CUDA in this
process, etc.) is fine — the copy still works, just slower."""
if self._pin_attempted:
return
self._pin_attempted = True
try:
if not current_platform.is_cuda_alike():
return
import ctypes
buf = self.shared_memory.buf
assert buf is not None
ptr = ctypes.addressof(ctypes.c_char.from_buffer(buf))
ret = current_platform.cudart().cudaHostRegister(ptr, self.total_bytes, 0)
self._pinned = int(ret) == 0
if self._pinned:
self._pinned_ptr = ptr
logger.info(
"ShmTensorArena: cudaHostRegister(%d MB) -> %s",
self.total_bytes >> 20,
"pinned" if self._pinned else f"error {int(ret)}",
)
except Exception as e:
logger.info("ShmTensorArena: host-register skipped: %s", e)
def _unpin(self):
"""Undo the pin from _ensure_pinned via cudaHostUnregister; must run
before the mapping is closed. Failures are ignored — at interpreter
shutdown the CUDA context may already be gone, and the registration
dies with the process anyway."""
if not self._pinned:
return
self._pinned = False
with suppress(Exception):
current_platform.cudart().cudaHostUnregister(self._pinned_ptr)
def get_tensor(
self, idx: int, nbytes: int, dtype: torch.dtype, shape: tuple[int, ...]
) -> torch.Tensor:
"""Zero-copy view of a slot as a tensor. The slot is queued for
release once the returned tensor's storage is freed, i.e. once the
caller has dropped it along with any views/slices or wrappers (such
as `torch.nn.Parameter`) sharing that storage."""
self._ensure_pinned()
# The tensor's storage holds the only reference to `holder` and drops
# it when freed, firing the finalizer. (`weakref.finalize` can't
# target the tensor itself, which views and wrappers don't keep alive,
# nor the memoryview, which doesn't support weakrefs.)
holder = np.frombuffer(self._slot(idx, nbytes), dtype=np.uint8)
# Binds to the current list OBJECT: `flush_releases` must mutate it in
# place, never rebind `self._pending_release`.
weakref.finalize(holder, self._pending_release.append, idx)
return torch.from_numpy(holder).view(dtype).view(shape)
def _mark_released(self, idxs: list[int]):
"""Set THIS reader's done flag on the given slots; a slot becomes
reusable once every reader has done so."""
for idx in idxs:
with self._meta(idx) as meta:
meta[1 + self.reader_rank] = 1
memory_fence()
def _record_release_event(self):
"""Record a CUDA event on the current (compute) stream, or return None
if this process has no usable CUDA context. Called from flush_releases,
which runs after the previous step's H2D was enqueued on that same
stream, so the event is ordered strictly after that H2D."""
try:
event = current_platform.Event()
event.record()
return event
except Exception:
return None
def flush_releases(self):
"""Retire this reader's arena slots that are ready for writer reuse.
Called on every `MessageQueue.dequeue`. `_pending_release` holds
slots whose tensor storage was just freed (see `get_tensor`). On
the pinned fast path that tensor was the source of an async
`non_blocking=True` H2D that can outlive the collection, so instead
of marking released immediately we record a CUDA event (ordered
after that H2D on the same stream) and move the slot to
`_deferred_releases` until `event.query()` says it's done -- a
not-yet-complete slot just waits one more `dequeue`. Unpinned (or no
CUDA context), the H2D already staged synchronously, so the slot is
safe to release right away.
A slot only becomes free for the writer (`write_tensor`) once every
reader has marked it released (`_mark_released`).
"""
if self._deferred_releases:
still_pending = []
for deferred_event, idxs in self._deferred_releases:
if deferred_event.query():
self._mark_released(idxs)
else:
still_pending.append((deferred_event, idxs))
self._deferred_releases = still_pending
if not self._pending_release:
return
# Drain in place with atomic pops, never rebind (see get_tensor): a
# finalizer may append from another thread concurrently.
idxs = []
while self._pending_release:
idxs.append(self._pending_release.pop())
event = self._record_release_event() if self._pinned else None
if event is None:
self._mark_released(idxs)
else:
self._deferred_releases.append((event, idxs))
def __del__(self):
if hasattr(self, "shared_memory"):
self._unpin()
try:
self.shared_memory.close()
if self.is_creator:
self.shared_memory.unlink()
except BufferError:
# zero-copy tensor views may still hold exported pointers at
# interpreter shutdown; the mapping dies with the process.
pass
def _rebuild_arena_tensor(arena_name, slot_idx, nbytes, dtype_str, shape):
"""Unpickle hook: rebuild a tensor as a zero-copy view of an arena slot."""
arena = _TENSOR_ARENAS[arena_name]
return arena.get_tensor(slot_idx, nbytes, getattr(torch, dtype_str), shape)
def _rebuild_arena_parameter(arena_name, slot_idx, nbytes, dtype_str, shape):
"""Unpickle hook for `torch.nn.Parameter` (shares the slot's storage)."""
t = _rebuild_arena_tensor(arena_name, slot_idx, nbytes, dtype_str, shape)
return torch.nn.Parameter(t, requires_grad=False)
# Types `_ArenaPickler.reducer_override` diverts, mapped to the rebuild hook
# that reconstructs them. Any other type -- including other `torch.Tensor`
# subclasses -- falls through to the normal pickle path, since an arbitrary
# subclass can't be faithfully rebuilt from just the slot's bytes.
_ARENA_REBUILD_FNS = {
torch.Tensor: _rebuild_arena_tensor,
torch.nn.Parameter: _rebuild_arena_parameter,
}
class _ArenaPickler(pickle.Pickler):
"""Pickler that diverts large contiguous CPU tensors into the arena.
`reducer_override` is consulted before an object's normal reduction. For
a qualifying tensor (exact type in `_ARENA_REBUILD_FNS`, CPU, strided,
contiguous, not `requires_grad`, above the divert-size threshold -- the
same criteria `_reduce_tensor` uses, plus size) it does one memcpy into a
free slot and returns a tiny rebuild stub. Anything that doesn't qualify,
or finds the arena full, returns `NotImplemented` and falls through to
the normal out-of-band pickle path, exactly as if no arena were attached.
"""
def __init__(self, file, arena: ShmTensorArena, buffer_callback=None):
super().__init__(
file,
protocol=pickle.HIGHEST_PROTOCOL,
buffer_callback=buffer_callback,
)
self.arena = arena
def reducer_override(self, obj):
rebuild_fn = _ARENA_REBUILD_FNS.get(type(obj))
if (
rebuild_fn is not None
and obj.device.type == "cpu"
and obj.layout == torch.strided
and obj.is_contiguous()
and not obj.requires_grad
and obj.numel() * obj.element_size() >= _ARENA_MIN_BYTES
):
idx = self.arena.write_tensor(obj)
if idx is not None:
return (
rebuild_fn,
(
self.arena.shared_memory.name,
idx,
obj.numel() * obj.element_size(),
str(obj.dtype).removeprefix("torch."),
tuple(obj.shape),
),
)
return NotImplemented
+6
View File
@@ -622,6 +622,7 @@ class EngineArgs:
allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization
enforce_eager: bool = ModelConfig.enforce_eager
disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
enable_shm_tensor_arena: bool = ParallelConfig.enable_shm_tensor_arena
language_model_only: bool = MultiModalConfig.language_model_only
limit_mm_per_prompt: dict[str, int | dict[str, int]] = get_field(
MultiModalConfig, "limit_per_prompt"
@@ -1274,6 +1275,10 @@ class EngineArgs:
"--all2all-backend", **parallel_kwargs["all2all_backend"]
)
parallel_group.add_argument("--enable-dbo", **parallel_kwargs["enable_dbo"])
parallel_group.add_argument(
"--enable-shm-tensor-arena",
**parallel_kwargs["enable_shm_tensor_arena"],
)
parallel_group.add_argument(
"--ubatch-size",
**parallel_kwargs["ubatch_size"],
@@ -2534,6 +2539,7 @@ class EngineArgs:
enable_elastic_ep=self.enable_elastic_ep,
elastic_ep_max_dp_size=self.elastic_ep_max_dp_size,
enable_dbo=self.enable_dbo,
enable_shm_tensor_arena=self.enable_shm_tensor_arena,
ubatch_size=self.ubatch_size,
dbo_decode_token_threshold=self.dbo_decode_token_threshold,
dp_sync_interval=self.dp_sync_interval,
+1
View File
@@ -166,6 +166,7 @@ class MultiprocExecutor(Executor):
self.local_world_size,
max_chunk_bytes=max_chunk_bytes,
connect_ip=mq_connect_ip,
enable_shm_tensor_arena=(self.parallel_config.enable_shm_tensor_arena),
)
scheduler_output_handle = self.rpc_broadcast_mq.export_handle()
# Create workers