[Core] shm tensor arena for efficient cpu->gpu worker broadcast (#51207)

Signed-off-by: Bolin Sun <bolins@nvidia.com>
Signed-off-by: <bolins@nvidia.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Bolin Sun
2026-10-09 16:13:24 -07:00
committed by GitHub
co-authored by Claude Nick Hill
parent fa94ad4143
commit 04b02fc8a5
8 changed files with 1290 additions and 8 deletions
+519 -3
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
import gc
import io
import pickle
@@ -17,7 +18,7 @@ import pytest
import torch
import torch.distributed as dist
from vllm.distributed.device_communicators import shm_broadcast
from vllm.distributed.device_communicators import shm_broadcast, shm_tensor_arena
from vllm.distributed.device_communicators.shm_broadcast import (
MessageQueue,
ShmRingBuffer,
@@ -25,6 +26,10 @@ from vllm.distributed.device_communicators.shm_broadcast import (
_reduce_tensor,
check_shm_free_space,
)
from vllm.distributed.device_communicators.shm_tensor_arena import (
ShmTensorArena,
_ArenaPickler,
)
from vllm.distributed.utils import StatelessProcessGroup
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
@@ -38,15 +43,18 @@ def get_arrays(n: int, seed: int = 0) -> list[np.ndarray]:
return [np.random.randint(1, 100, i) for i in sizes]
def distributed_run(fn, world_size, timeout=60):
def distributed_run(fn, world_size, timeout=60, mp_context=None):
"""Run a function in multiple processes with proper error handling.
Args:
fn: Function to run in each process
world_size: Number of processes to spawn
timeout: Maximum time in seconds to wait for processes (default: 60)
mp_context: Optional multiprocess context (e.g. ``mp.get_context("spawn")``)
for workers that must not inherit the parent's thread/CUDA state.
"""
ctx = mp_context or mp
number_of_processes = world_size
processes = []
for i in range(number_of_processes):
@@ -57,7 +65,7 @@ def distributed_run(fn, world_size, timeout=60):
env["LOCAL_WORLD_SIZE"] = str(number_of_processes)
env["MASTER_ADDR"] = "localhost"
env["MASTER_PORT"] = "12345"
p = mp.Process(target=fn, args=(env,))
p = ctx.Process(target=fn, args=(env,))
processes.append(p)
p.start()
@@ -765,6 +773,20 @@ def test_shm_ring_buffer_creation_checks_free_space():
ShmRingBuffer(n_reader=1, max_chunk_bytes=24 * 1024 * 1024, max_chunks=10)
def test_shm_tensor_arena_creation_checks_free_space():
"""Like ShmRingBuffer, arena creation must fail fast with a clear error
on an undersized /dev/shm instead of discovering the shortfall via a
SIGBUS the first time a tensor is copied into pages past tmpfs capacity."""
with (
mock.patch.object(
shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(1 << 20)
),
mock.patch.object(shm_broadcast.os.path, "isdir", return_value=True),
pytest.raises(RuntimeError, match="Insufficient space"),
):
ShmTensorArena(n_reader=1, slot_bytes=256 << 20, n_slots=8)
def test_remote_subscribe_addr_unique_concurrent_writers(
monkeypatch: pytest.MonkeyPatch,
):
@@ -828,3 +850,497 @@ def test_remote_subscribe_addr_unique_concurrent_writers(
for q in queues:
q.remote_socket.close(linger=0)
# --------------- ShmTensorArena (zero-copy tensor arena) tests ---------------
def _make_arena(n_reader: int = 2, slot_bytes: int = 4 << 20, n_slots: int = 3):
"""Writer arena plus attached per-reader arenas (same process)."""
writer = ShmTensorArena(n_reader, slot_bytes, n_slots)
readers = [ShmTensorArena(*writer.handle(), reader_rank=i) for i in range(n_reader)]
return writer, readers
def _get_view(reader: ShmTensorArena, idx: int, ref: torch.Tensor) -> torch.Tensor:
return reader.get_tensor(
idx, ref.numel() * ref.element_size(), ref.dtype, tuple(ref.shape)
)
def _drain(reader: ShmTensorArena) -> None:
"""Flush a reader's releases to completion. On the pinned path the first
flush defers the release behind an H2D-completion event; the second flush
retires it once the event has fired."""
reader.flush_releases()
if reader._deferred_releases:
torch.accelerator.synchronize()
reader.flush_releases()
assert not reader._pending_release
assert not reader._deferred_releases
def test_arena_zero_copy_roundtrip():
writer, (r0, r1) = _make_arena(n_reader=2)
src = torch.randn(400, 512)
idx = writer.write_tensor(src)
assert idx is not None
a = _get_view(r0, idx, src)
b = _get_view(r1, idx, src)
assert torch.equal(a, src)
assert torch.equal(b, src)
assert a.dtype == src.dtype and a.shape == src.shape
# Readers alias one shared mapping: a write through one reader's view is
# visible through the other's (this is what "zero-copy" means here).
a[0, 0] = 12345.0
assert b[0, 0].item() == 12345.0
del a, b
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
def test_arena_dtype_roundtrip(dtype):
"""Dtypes numpy doesn't recognize traverse the arena's uint8 view path."""
writer, (reader,) = _make_arena(n_reader=1)
src = torch.randn(256, 256).to(dtype)
idx = writer.write_tensor(src)
assert idx is not None
got = _get_view(reader, idx, src)
assert got.dtype == dtype
assert torch.equal(got.view(torch.uint8), src.view(torch.uint8))
del got
def test_arena_slot_lifecycle():
"""A slot must not be reusable until EVERY reader has released it —
the writer overwriting a slot a reader still consumes would corrupt data.
Release is tied to the returned tensor's storage lifetime (`get_tensor`):
a reader that still holds its view keeps the slot reserved even across
repeated `flush_releases()` calls, not just until the "next" one."""
writer, readers = _make_arena(n_reader=2, n_slots=3)
t = torch.ones(1000)
first = writer.write_tensor(t)
views = [_get_view(r, first, t) for r in readers]
# Fill the remaining slots; the arena is now exhausted.
assert all(writer.write_tensor(t) is not None for _ in range(2))
assert writer.write_tensor(t) is None # exhausted -> caller falls back
# Flushing while both readers still hold their view releases nothing:
# the slot is only queued once the returned tensor's storage is freed.
readers[0].flush_releases()
readers[1].flush_releases()
assert writer.write_tensor(t) is None
# Dropping one reader's view queues its release; still not enough.
views[0] = None
_drain(readers[0])
assert writer.write_tensor(t) is None
# Once every reader has dropped (and released) its view, it's reused.
views[1] = None
_drain(readers[1])
assert writer.write_tensor(t) == first
def test_arena_slot_not_released_while_tensor_retained_across_multiple_flushes():
"""A caller retaining the tensor across many `flush_releases()` calls
(e.g. `prompt_embeds` across chunked-prefill steps) must keep its slot
reserved the whole time, not just until the reader's next flush."""
writer, (reader,) = _make_arena(n_reader=1, n_slots=2)
t = torch.ones(1000)
idx = writer.write_tensor(t)
retained = _get_view(reader, idx, t)
# Several flush cycles pass while the caller still holds `retained`.
for _ in range(5):
reader.flush_releases()
assert reader._pending_release == []
# The slot is still reserved: only the arena's other slot is available...
assert writer.write_tensor(t) is not None
assert writer.write_tensor(t) is None # ...and now the arena is exhausted.
# ...and the bytes the caller is reading are unchanged throughout.
assert torch.equal(retained, t)
# Only once the caller drops its reference is the slot released.
del retained
_drain(reader)
assert writer.write_tensor(t) == idx
def test_arena_slot_not_released_while_view_retained():
"""A view/slice (or wrapper) sharing the returned tensor's storage keeps
the slot reserved even after the original tensor object is dropped."""
writer, (reader,) = _make_arena(n_reader=1, n_slots=1)
t = torch.arange(1000, dtype=torch.float32)
idx = writer.write_tensor(t)
view = _get_view(reader, idx, t)[10:20].unsqueeze(0)
param = torch.nn.Parameter(view, requires_grad=False)
del view
reader.flush_releases()
assert reader._pending_release == []
assert writer.write_tensor(t) is None
assert torch.equal(param[0], t[10:20])
del param
assert reader._pending_release == [idx]
_drain(reader)
assert writer.write_tensor(t) == idx
def test_arena_release_survives_intervening_flush_from_another_tensor():
"""`flush_releases()` must clear `_pending_release` in place, never
rebind it: a finalizer registered before a rebind would append into the
now-orphaned old list and permanently leak its slot. Exercises the
interleaving a long-lived `prompt_embeds` tensor sees once a
shorter-lived tensor through the same reader triggers a flush first."""
writer, (reader,) = _make_arena(n_reader=1, n_slots=2)
t = torch.ones(1000)
# `a` is long-lived: its finalizer is registered first, while
# _pending_release is still the arena's original (empty) list object.
idx_a = writer.write_tensor(t)
a = _get_view(reader, idx_a, t)
# `b` is short-lived and its finalizer is registered against that SAME
# list object (nothing has flushed yet).
idx_b = writer.write_tensor(t)
b = _get_view(reader, idx_b, t)
del b
assert reader._pending_release == [idx_b]
# This flush must not orphan `a`'s still-pending finalizer. `_drain`,
# not a bare `flush_releases()` call: on the pinned fast path (real
# CUDA present) the release is gated behind an event and may need a
# second flush once it fires -- see `flush_releases`.
_drain(reader)
assert writer.write_tensor(t) == idx_b # b's slot correctly reused.
# `a` is dropped well after that intervening flush.
del a
assert reader._pending_release == [idx_a]
_drain(reader)
assert writer.write_tensor(t) == idx_a # a's slot must NOT be leaked.
def test_arena_oversize_falls_back():
writer, _ = _make_arena(n_reader=1, slot_bytes=1 << 20, n_slots=2)
big = torch.empty((1 << 20) + 4096, dtype=torch.uint8)
assert writer.write_tensor(big) is None
def _dumps_arena(obj, arena: ShmTensorArena) -> tuple[bytes, list, list[int]]:
"""Pickle `obj` the same way `MessageQueue.enqueue` does when an arena is
attached: arena diversion first (reducer_override), then the tensor
dispatch table, with out-of-band buffers >= 1MiB. Also returns the slot
indices the arena actually wrote to, in write order."""
buffers = []
def callback(buf: pickle.PickleBuffer) -> bool:
raw = buf.raw()
if raw.nbytes < 1024 * 1024:
return True
buffers.append(raw)
return False
written: list[int] = []
orig_write_tensor = arena.write_tensor
def _spy_write_tensor(t):
idx = orig_write_tensor(t)
if idx is not None:
written.append(idx)
return idx
bio = io.BytesIO()
pickler = _ArenaPickler(bio, arena, buffer_callback=callback)
pickler.dispatch_table = {torch.Tensor: _reduce_tensor}
with mock.patch.object(arena, "write_tensor", side_effect=_spy_write_tensor):
pickler.dump(obj)
return bio.getvalue(), buffers, written
def test_arena_pickler_composes(monkeypatch):
"""Large contiguous tensors are diverted into the arena; everything the
arena declines falls through to `_reduce_tensor` unchanged."""
writer, (reader,) = _make_arena(n_reader=1)
monkeypatch.setattr(shm_tensor_arena, "_ARENA_MIN_BYTES", 1 << 20)
monkeypatch.setitem(
shm_tensor_arena._TENSOR_ARENAS, reader.shared_memory.name, reader
)
big = torch.randn(1024, 1024) # 4MiB -> diverted into the arena
small = torch.randn(16, 16) # 1KiB -> falls through to _reduce_tensor
data, buffers, written = _dumps_arena({"big": big, "small": small}, writer)
# The diverted tensor's bytes are in the arena, not the pickle stream.
assert len(data) + sum(b.nbytes for b in buffers) < big.numel() * 4
out = pickle.loads(data, buffers=buffers)
assert torch.equal(out["big"], big)
assert torch.equal(out["small"], small)
# "big" is a zero-copy view of the reader's slot; "small" is not.
(idx,) = written
nbytes = big.numel() * big.element_size()
slot_ptr = torch.frombuffer(reader._slot(idx, nbytes), dtype=torch.uint8).data_ptr()
assert out["big"].data_ptr() == slot_ptr
assert out["small"].data_ptr() != slot_ptr
# Not yet queued for release: `out["big"]` is still alive (see get_tensor).
assert reader._pending_release == []
del out
assert reader._pending_release == [idx]
_drain(reader)
def test_arena_pickler_noncontig_falls_through(monkeypatch):
writer, (reader,) = _make_arena(n_reader=1)
monkeypatch.setattr(shm_tensor_arena, "_ARENA_MIN_BYTES", 1 << 20)
monkeypatch.setitem(
shm_tensor_arena._TENSOR_ARENAS, reader.shared_memory.name, reader
)
nc = torch.randn(2048, 1024)[:, ::2] # non-contiguous, above threshold
assert not nc.is_contiguous()
data, buffers, written = _dumps_arena(nc, writer)
out = pickle.loads(data, buffers=buffers)
assert torch.equal(out, nc)
# The arena never saw it: no slot consumed on the reader.
assert written == []
assert reader._pending_release == []
del out
@pytest.mark.parametrize("case", ["requires_grad", "conj"])
def test_arena_pickler_excludes_requires_grad_and_conj(monkeypatch, case: str):
"""Tensors `_reduce_tensor` excludes (requires_grad, or an exotic
conjugate-bit view that can't alias as uint8) must fall through the
arena the same way, even when every other criterion qualifies."""
writer, (reader,) = _make_arena(n_reader=1)
monkeypatch.setattr(shm_tensor_arena, "_ARENA_MIN_BYTES", 1 << 20)
monkeypatch.setitem(
shm_tensor_arena._TENSOR_ARENAS, reader.shared_memory.name, reader
)
if case == "requires_grad":
tensor = torch.randn(1024, 1024, requires_grad=True) # 4MiB, above threshold
else:
tensor = torch.randn(600, 600, dtype=torch.complex64).conj() # ~2.9MiB
assert tensor.is_contiguous()
assert tensor.numel() * tensor.element_size() >= shm_tensor_arena._ARENA_MIN_BYTES
data, buffers, written = _dumps_arena(tensor, writer)
# Declined by the arena -- no slot consumed -- and handled by
# _reduce_tensor's identical exclusion instead.
assert written == []
out = pickle.loads(data, buffers=buffers)
assert torch.equal(out, tensor)
assert out.requires_grad == tensor.requires_grad
def test_arena_pickler_excludes_tensor_subclasses(monkeypatch):
"""`torch.nn.Parameter` is diverted via its own `_rebuild_arena_parameter`
hook, which wraps an intermediate tensor it keeps no Python reference to.
The slot must stay reserved while the Parameter is alive -- the
`_pending_release` assertions below check this even though
`type(out) is torch.nn.Parameter` alone would not."""
writer, (reader,) = _make_arena(n_reader=1)
monkeypatch.setattr(shm_tensor_arena, "_ARENA_MIN_BYTES", 1 << 20)
monkeypatch.setitem(
shm_tensor_arena._TENSOR_ARENAS, reader.shared_memory.name, reader
)
# requires_grad=False to isolate the subclass check from the (already
# separately tested) requires_grad exclusion.
param = torch.nn.Parameter(torch.randn(1024, 1024), requires_grad=False)
assert type(param) is not torch.Tensor
assert param.is_contiguous()
assert param.numel() * param.element_size() >= shm_tensor_arena._ARENA_MIN_BYTES
data, buffers, written = _dumps_arena(param, writer)
# The Parameter itself is diverted directly (_ARENA_REBUILD_FNS routes
# it to _rebuild_arena_parameter) -- Parameter.__reduce_ex__ never runs,
# so there's no separate "inner" tensor diversion to speak of.
assert written != []
out = pickle.loads(data, buffers=buffers)
# The outer Parameter's type identity survives.
assert type(out) is torch.nn.Parameter
assert torch.equal(out, param)
assert out.requires_grad == param.requires_grad
# Not yet queued for release: `out` still shares the slot's storage.
assert reader._pending_release == []
del out
assert reader._pending_release == written
_drain(reader)
def test_arena_unregisters_pinned_memory_on_del(monkeypatch):
"""A reader that cudaHostRegister-ed its mapping must cudaHostUnregister
the same pointer before the mapping is closed; an unpinned arena must not
unregister memory it never registered."""
writer, (reader,) = _make_arena(n_reader=1)
cudart = mock.MagicMock()
cudart.cudaHostRegister.return_value = 0
cudart.cudaHostUnregister.return_value = 0
monkeypatch.setattr(
shm_tensor_arena,
"current_platform",
SimpleNamespace(is_cuda_alike=lambda: True, cudart=lambda: cudart),
)
reader._ensure_pinned()
assert reader._pinned
ptr = cudart.cudaHostRegister.call_args.args[0]
del reader
cudart.cudaHostUnregister.assert_called_once_with(ptr)
del writer # never pinned -> no unregister
cudart.cudaHostUnregister.assert_called_once()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
def test_arena_event_gated_release():
"""On the pinned path a slot release is gated on an H2D-completion CUDA
event: the writer must not see the reader's done flag until the async DMA
sourced from the slot has been retired. Release is first queued once the
caller drops the CPU-side view -- the realistic pattern: the view is only
needed as the H2D source, while the GPU tensor is what's retained for the
rest of the step."""
writer, (reader,) = _make_arena(n_reader=1)
reader._ensure_pinned()
if not reader._pinned:
pytest.skip("cudaHostRegister unavailable in this environment")
src = torch.randn(512, 1024)
idx = writer.write_tensor(src)
view = _get_view(reader, idx, src)
dev = view.to("cuda", non_blocking=True)
# Not yet queued: this test still holds `view`.
assert reader._pending_release == []
del view
assert reader._pending_release == [idx]
reader.flush_releases()
# Deferred behind the event, not applied eagerly.
assert len(reader._deferred_releases) == 1
with reader._meta(idx) as meta:
assert meta[1] == 0
torch.accelerator.synchronize()
reader.flush_releases()
assert reader._deferred_releases == []
with reader._meta(idx) as meta:
assert meta[1] == 1
assert torch.equal(dev.cpu(), src)
del dev
def test_tensor_arena_registry_is_weak_and_survives_shutdown():
"""`_TENSOR_ARENAS` must not keep an arena alive past its owning
MessageQueue, but also must not be cleared by `shutdown()` -- which can
run on a different thread than an in-flight `dequeue()` still
reconstructing a tensor from that arena. A weak-value registry gets
both: the entry survives `shutdown()` and vanishes only once
unreferenced."""
writer_mq = MessageQueue(n_reader=1, n_local_reader=1, enable_shm_tensor_arena=True)
handle = writer_mq.export_handle()
reader_mq = MessageQueue.create_from_handle(handle, rank=0)
name = reader_mq.tensor_arena.shared_memory.name
assert shm_tensor_arena._TENSOR_ARENAS[name] is reader_mq.tensor_arena
# shutdown() must NOT remove the entry: that would race with a
# concurrent dequeue() still reconstructing a tensor from this arena.
reader_mq.shutdown()
assert shm_tensor_arena._TENSOR_ARENAS[name] is reader_mq.tensor_arena
# Once nothing else references the arena, the registry entry vanishes
# on its own.
reader_mq.tensor_arena = None
assert name not in shm_tensor_arena._TENSOR_ARENAS
writer_mq.shutdown()
def test_create_from_handle_tolerates_arena_attach_failure():
"""If a reader's `ShmTensorArena.__init__` can't attach to the writer's
segment (`FileNotFoundError`, e.g. deserialized on a different node),
`create_from_handle` must not crash -- same deferred-failure convention
as the sibling `ShmRingBuffer` path."""
writer_mq = MessageQueue(n_reader=1, n_local_reader=1, enable_shm_tensor_arena=True)
handle = writer_mq.export_handle()
n_reader, slot_bytes, n_slots, _real_name = handle.tensor_arena_handle
bogus_handle = dataclasses.replace(
handle,
tensor_arena_handle=(
n_reader,
slot_bytes,
n_slots,
"vllm-arena-does-not-exist",
),
)
reader_mq = MessageQueue.create_from_handle(bogus_handle, rank=0)
# Attach failed silently: the arena object exists but never got its
# shared_memory attribute, and was never registered.
assert reader_mq.tensor_arena is not None
assert not hasattr(reader_mq.tensor_arena, "shared_memory")
assert "vllm-arena-does-not-exist" not in shm_tensor_arena._TENSOR_ARENAS
reader_mq.shutdown()
writer_mq.shutdown()
@worker_fn_wrapper
def worker_fn_arena_broadcast():
rank = dist.get_rank()
writer_rank = 0
if rank == writer_rank:
message_queue = MessageQueue(
1,
1,
local_reader_ranks=[1],
max_chunk_bytes=8 * 1024 * 1024,
enable_shm_tensor_arena=True,
)
handles = [message_queue.export_handle()]
dist.broadcast_object_list(handles, src=writer_rank)
else:
handles = [None]
dist.broadcast_object_list(handles, src=writer_rank)
message_queue = MessageQueue.create_from_handle(handles[0], rank)
message_queue.wait_until_ready()
torch.manual_seed(42)
payload = {
# 16MiB: above the arena divert threshold (8MiB) -> arena slot.
"huge": torch.randn(2048, 2048),
# 2MiB: declined by the arena -> out-of-band _reduce_tensor path.
"mid": torch.randn(1024, 512),
}
if rank == writer_rank:
assert message_queue.tensor_arena is not None
with mock.patch(
"vllm.distributed.device_communicators.shm_broadcast._reduce_tensor",
wraps=_reduce_tensor,
) as wrapped_reduce:
message_queue.enqueue(payload)
# "huge" was diverted into the arena before the dispatch table was
# consulted; only "mid" went through _reduce_tensor.
assert wrapped_reduce.call_count == 1
else:
received = message_queue.dequeue(timeout=30)
assert torch.equal(received["huge"], payload["huge"])
assert torch.equal(received["mid"], payload["mid"])
# The huge tensor is a zero-copy view into the arena's shared memory
# (not a copy received over the transport). Its slot is not yet
# queued for release while `received` still holds the reference.
arena = message_queue.tensor_arena
assert arena is not None
assert arena._pending_release == []
huge_ptr = received["huge"].data_ptr()
arena_base = torch.frombuffer(
arena.shared_memory.buf, dtype=torch.uint8
).data_ptr()
assert arena_base <= huge_ptr < arena_base + arena.total_bytes
# Dropping the last reference to the slot's storage queues it for
# release (see get_tensor).
del received
assert len(arena._pending_release) == 1
dist.barrier()
print(f"arena broadcast passed the test! Rank {rank}")
def test_arena_broadcast():
# Spawn (not fork): by the time this test runs, the pytest process may be
# multi-threaded / CUDA-initialized (earlier GPU tests), and a forked
# child running the arena's slot memcpy can deadlock on inherited lock
# state. Spawned workers start clean; the larger timeout absorbs their
# interpreter + import startup.
distributed_run(
worker_fn_arena_broadcast,
2,
timeout=180,
mp_context=mp.get_context("spawn"),
)