[Core] shm tensor arena for efficient cpu->gpu worker broadcast (#51207)
Signed-off-by: Bolin Sun <bolins@nvidia.com> Signed-off-by: <bolins@nvidia.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude
Nick Hill
parent
fa94ad4143
commit
04b02fc8a5
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import dataclasses
|
||||
import gc
|
||||
import io
|
||||
import pickle
|
||||
@@ -17,7 +18,7 @@ import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.distributed.device_communicators import shm_broadcast
|
||||
from vllm.distributed.device_communicators import shm_broadcast, shm_tensor_arena
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
MessageQueue,
|
||||
ShmRingBuffer,
|
||||
@@ -25,6 +26,10 @@ from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
_reduce_tensor,
|
||||
check_shm_free_space,
|
||||
)
|
||||
from vllm.distributed.device_communicators.shm_tensor_arena import (
|
||||
ShmTensorArena,
|
||||
_ArenaPickler,
|
||||
)
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
@@ -38,15 +43,18 @@ def get_arrays(n: int, seed: int = 0) -> list[np.ndarray]:
|
||||
return [np.random.randint(1, 100, i) for i in sizes]
|
||||
|
||||
|
||||
def distributed_run(fn, world_size, timeout=60):
|
||||
def distributed_run(fn, world_size, timeout=60, mp_context=None):
|
||||
"""Run a function in multiple processes with proper error handling.
|
||||
|
||||
Args:
|
||||
fn: Function to run in each process
|
||||
world_size: Number of processes to spawn
|
||||
timeout: Maximum time in seconds to wait for processes (default: 60)
|
||||
mp_context: Optional multiprocess context (e.g. ``mp.get_context("spawn")``)
|
||||
for workers that must not inherit the parent's thread/CUDA state.
|
||||
|
||||
"""
|
||||
ctx = mp_context or mp
|
||||
number_of_processes = world_size
|
||||
processes = []
|
||||
for i in range(number_of_processes):
|
||||
@@ -57,7 +65,7 @@ def distributed_run(fn, world_size, timeout=60):
|
||||
env["LOCAL_WORLD_SIZE"] = str(number_of_processes)
|
||||
env["MASTER_ADDR"] = "localhost"
|
||||
env["MASTER_PORT"] = "12345"
|
||||
p = mp.Process(target=fn, args=(env,))
|
||||
p = ctx.Process(target=fn, args=(env,))
|
||||
processes.append(p)
|
||||
p.start()
|
||||
|
||||
@@ -765,6 +773,20 @@ def test_shm_ring_buffer_creation_checks_free_space():
|
||||
ShmRingBuffer(n_reader=1, max_chunk_bytes=24 * 1024 * 1024, max_chunks=10)
|
||||
|
||||
|
||||
def test_shm_tensor_arena_creation_checks_free_space():
|
||||
"""Like ShmRingBuffer, arena creation must fail fast with a clear error
|
||||
on an undersized /dev/shm instead of discovering the shortfall via a
|
||||
SIGBUS the first time a tensor is copied into pages past tmpfs capacity."""
|
||||
with (
|
||||
mock.patch.object(
|
||||
shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(1 << 20)
|
||||
),
|
||||
mock.patch.object(shm_broadcast.os.path, "isdir", return_value=True),
|
||||
pytest.raises(RuntimeError, match="Insufficient space"),
|
||||
):
|
||||
ShmTensorArena(n_reader=1, slot_bytes=256 << 20, n_slots=8)
|
||||
|
||||
|
||||
def test_remote_subscribe_addr_unique_concurrent_writers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
@@ -828,3 +850,497 @@ def test_remote_subscribe_addr_unique_concurrent_writers(
|
||||
|
||||
for q in queues:
|
||||
q.remote_socket.close(linger=0)
|
||||
|
||||
|
||||
# --------------- ShmTensorArena (zero-copy tensor arena) tests ---------------
|
||||
|
||||
|
||||
def _make_arena(n_reader: int = 2, slot_bytes: int = 4 << 20, n_slots: int = 3):
|
||||
"""Writer arena plus attached per-reader arenas (same process)."""
|
||||
writer = ShmTensorArena(n_reader, slot_bytes, n_slots)
|
||||
readers = [ShmTensorArena(*writer.handle(), reader_rank=i) for i in range(n_reader)]
|
||||
return writer, readers
|
||||
|
||||
|
||||
def _get_view(reader: ShmTensorArena, idx: int, ref: torch.Tensor) -> torch.Tensor:
|
||||
return reader.get_tensor(
|
||||
idx, ref.numel() * ref.element_size(), ref.dtype, tuple(ref.shape)
|
||||
)
|
||||
|
||||
|
||||
def _drain(reader: ShmTensorArena) -> None:
|
||||
"""Flush a reader's releases to completion. On the pinned path the first
|
||||
flush defers the release behind an H2D-completion event; the second flush
|
||||
retires it once the event has fired."""
|
||||
reader.flush_releases()
|
||||
if reader._deferred_releases:
|
||||
torch.accelerator.synchronize()
|
||||
reader.flush_releases()
|
||||
assert not reader._pending_release
|
||||
assert not reader._deferred_releases
|
||||
|
||||
|
||||
def test_arena_zero_copy_roundtrip():
|
||||
writer, (r0, r1) = _make_arena(n_reader=2)
|
||||
src = torch.randn(400, 512)
|
||||
idx = writer.write_tensor(src)
|
||||
assert idx is not None
|
||||
a = _get_view(r0, idx, src)
|
||||
b = _get_view(r1, idx, src)
|
||||
assert torch.equal(a, src)
|
||||
assert torch.equal(b, src)
|
||||
assert a.dtype == src.dtype and a.shape == src.shape
|
||||
# Readers alias one shared mapping: a write through one reader's view is
|
||||
# visible through the other's (this is what "zero-copy" means here).
|
||||
a[0, 0] = 12345.0
|
||||
assert b[0, 0].item() == 12345.0
|
||||
del a, b
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
def test_arena_dtype_roundtrip(dtype):
|
||||
"""Dtypes numpy doesn't recognize traverse the arena's uint8 view path."""
|
||||
writer, (reader,) = _make_arena(n_reader=1)
|
||||
src = torch.randn(256, 256).to(dtype)
|
||||
idx = writer.write_tensor(src)
|
||||
assert idx is not None
|
||||
got = _get_view(reader, idx, src)
|
||||
assert got.dtype == dtype
|
||||
assert torch.equal(got.view(torch.uint8), src.view(torch.uint8))
|
||||
del got
|
||||
|
||||
|
||||
def test_arena_slot_lifecycle():
|
||||
"""A slot must not be reusable until EVERY reader has released it —
|
||||
the writer overwriting a slot a reader still consumes would corrupt data.
|
||||
Release is tied to the returned tensor's storage lifetime (`get_tensor`):
|
||||
a reader that still holds its view keeps the slot reserved even across
|
||||
repeated `flush_releases()` calls, not just until the "next" one."""
|
||||
writer, readers = _make_arena(n_reader=2, n_slots=3)
|
||||
t = torch.ones(1000)
|
||||
first = writer.write_tensor(t)
|
||||
views = [_get_view(r, first, t) for r in readers]
|
||||
# Fill the remaining slots; the arena is now exhausted.
|
||||
assert all(writer.write_tensor(t) is not None for _ in range(2))
|
||||
assert writer.write_tensor(t) is None # exhausted -> caller falls back
|
||||
|
||||
# Flushing while both readers still hold their view releases nothing:
|
||||
# the slot is only queued once the returned tensor's storage is freed.
|
||||
readers[0].flush_releases()
|
||||
readers[1].flush_releases()
|
||||
assert writer.write_tensor(t) is None
|
||||
|
||||
# Dropping one reader's view queues its release; still not enough.
|
||||
views[0] = None
|
||||
_drain(readers[0])
|
||||
assert writer.write_tensor(t) is None
|
||||
# Once every reader has dropped (and released) its view, it's reused.
|
||||
views[1] = None
|
||||
_drain(readers[1])
|
||||
assert writer.write_tensor(t) == first
|
||||
|
||||
|
||||
def test_arena_slot_not_released_while_tensor_retained_across_multiple_flushes():
|
||||
"""A caller retaining the tensor across many `flush_releases()` calls
|
||||
(e.g. `prompt_embeds` across chunked-prefill steps) must keep its slot
|
||||
reserved the whole time, not just until the reader's next flush."""
|
||||
writer, (reader,) = _make_arena(n_reader=1, n_slots=2)
|
||||
t = torch.ones(1000)
|
||||
idx = writer.write_tensor(t)
|
||||
retained = _get_view(reader, idx, t)
|
||||
# Several flush cycles pass while the caller still holds `retained`.
|
||||
for _ in range(5):
|
||||
reader.flush_releases()
|
||||
assert reader._pending_release == []
|
||||
# The slot is still reserved: only the arena's other slot is available...
|
||||
assert writer.write_tensor(t) is not None
|
||||
assert writer.write_tensor(t) is None # ...and now the arena is exhausted.
|
||||
# ...and the bytes the caller is reading are unchanged throughout.
|
||||
assert torch.equal(retained, t)
|
||||
# Only once the caller drops its reference is the slot released.
|
||||
del retained
|
||||
_drain(reader)
|
||||
assert writer.write_tensor(t) == idx
|
||||
|
||||
|
||||
def test_arena_slot_not_released_while_view_retained():
|
||||
"""A view/slice (or wrapper) sharing the returned tensor's storage keeps
|
||||
the slot reserved even after the original tensor object is dropped."""
|
||||
writer, (reader,) = _make_arena(n_reader=1, n_slots=1)
|
||||
t = torch.arange(1000, dtype=torch.float32)
|
||||
idx = writer.write_tensor(t)
|
||||
view = _get_view(reader, idx, t)[10:20].unsqueeze(0)
|
||||
param = torch.nn.Parameter(view, requires_grad=False)
|
||||
del view
|
||||
reader.flush_releases()
|
||||
assert reader._pending_release == []
|
||||
assert writer.write_tensor(t) is None
|
||||
assert torch.equal(param[0], t[10:20])
|
||||
del param
|
||||
assert reader._pending_release == [idx]
|
||||
_drain(reader)
|
||||
assert writer.write_tensor(t) == idx
|
||||
|
||||
|
||||
def test_arena_release_survives_intervening_flush_from_another_tensor():
|
||||
"""`flush_releases()` must clear `_pending_release` in place, never
|
||||
rebind it: a finalizer registered before a rebind would append into the
|
||||
now-orphaned old list and permanently leak its slot. Exercises the
|
||||
interleaving a long-lived `prompt_embeds` tensor sees once a
|
||||
shorter-lived tensor through the same reader triggers a flush first."""
|
||||
writer, (reader,) = _make_arena(n_reader=1, n_slots=2)
|
||||
t = torch.ones(1000)
|
||||
# `a` is long-lived: its finalizer is registered first, while
|
||||
# _pending_release is still the arena's original (empty) list object.
|
||||
idx_a = writer.write_tensor(t)
|
||||
a = _get_view(reader, idx_a, t)
|
||||
# `b` is short-lived and its finalizer is registered against that SAME
|
||||
# list object (nothing has flushed yet).
|
||||
idx_b = writer.write_tensor(t)
|
||||
b = _get_view(reader, idx_b, t)
|
||||
del b
|
||||
assert reader._pending_release == [idx_b]
|
||||
# This flush must not orphan `a`'s still-pending finalizer. `_drain`,
|
||||
# not a bare `flush_releases()` call: on the pinned fast path (real
|
||||
# CUDA present) the release is gated behind an event and may need a
|
||||
# second flush once it fires -- see `flush_releases`.
|
||||
_drain(reader)
|
||||
assert writer.write_tensor(t) == idx_b # b's slot correctly reused.
|
||||
# `a` is dropped well after that intervening flush.
|
||||
del a
|
||||
assert reader._pending_release == [idx_a]
|
||||
_drain(reader)
|
||||
assert writer.write_tensor(t) == idx_a # a's slot must NOT be leaked.
|
||||
|
||||
|
||||
def test_arena_oversize_falls_back():
|
||||
writer, _ = _make_arena(n_reader=1, slot_bytes=1 << 20, n_slots=2)
|
||||
big = torch.empty((1 << 20) + 4096, dtype=torch.uint8)
|
||||
assert writer.write_tensor(big) is None
|
||||
|
||||
|
||||
def _dumps_arena(obj, arena: ShmTensorArena) -> tuple[bytes, list, list[int]]:
|
||||
"""Pickle `obj` the same way `MessageQueue.enqueue` does when an arena is
|
||||
attached: arena diversion first (reducer_override), then the tensor
|
||||
dispatch table, with out-of-band buffers >= 1MiB. Also returns the slot
|
||||
indices the arena actually wrote to, in write order."""
|
||||
buffers = []
|
||||
|
||||
def callback(buf: pickle.PickleBuffer) -> bool:
|
||||
raw = buf.raw()
|
||||
if raw.nbytes < 1024 * 1024:
|
||||
return True
|
||||
buffers.append(raw)
|
||||
return False
|
||||
|
||||
written: list[int] = []
|
||||
orig_write_tensor = arena.write_tensor
|
||||
|
||||
def _spy_write_tensor(t):
|
||||
idx = orig_write_tensor(t)
|
||||
if idx is not None:
|
||||
written.append(idx)
|
||||
return idx
|
||||
|
||||
bio = io.BytesIO()
|
||||
pickler = _ArenaPickler(bio, arena, buffer_callback=callback)
|
||||
pickler.dispatch_table = {torch.Tensor: _reduce_tensor}
|
||||
with mock.patch.object(arena, "write_tensor", side_effect=_spy_write_tensor):
|
||||
pickler.dump(obj)
|
||||
return bio.getvalue(), buffers, written
|
||||
|
||||
|
||||
def test_arena_pickler_composes(monkeypatch):
|
||||
"""Large contiguous tensors are diverted into the arena; everything the
|
||||
arena declines falls through to `_reduce_tensor` unchanged."""
|
||||
writer, (reader,) = _make_arena(n_reader=1)
|
||||
monkeypatch.setattr(shm_tensor_arena, "_ARENA_MIN_BYTES", 1 << 20)
|
||||
monkeypatch.setitem(
|
||||
shm_tensor_arena._TENSOR_ARENAS, reader.shared_memory.name, reader
|
||||
)
|
||||
big = torch.randn(1024, 1024) # 4MiB -> diverted into the arena
|
||||
small = torch.randn(16, 16) # 1KiB -> falls through to _reduce_tensor
|
||||
data, buffers, written = _dumps_arena({"big": big, "small": small}, writer)
|
||||
# The diverted tensor's bytes are in the arena, not the pickle stream.
|
||||
assert len(data) + sum(b.nbytes for b in buffers) < big.numel() * 4
|
||||
out = pickle.loads(data, buffers=buffers)
|
||||
assert torch.equal(out["big"], big)
|
||||
assert torch.equal(out["small"], small)
|
||||
# "big" is a zero-copy view of the reader's slot; "small" is not.
|
||||
(idx,) = written
|
||||
nbytes = big.numel() * big.element_size()
|
||||
slot_ptr = torch.frombuffer(reader._slot(idx, nbytes), dtype=torch.uint8).data_ptr()
|
||||
assert out["big"].data_ptr() == slot_ptr
|
||||
assert out["small"].data_ptr() != slot_ptr
|
||||
# Not yet queued for release: `out["big"]` is still alive (see get_tensor).
|
||||
assert reader._pending_release == []
|
||||
del out
|
||||
assert reader._pending_release == [idx]
|
||||
_drain(reader)
|
||||
|
||||
|
||||
def test_arena_pickler_noncontig_falls_through(monkeypatch):
|
||||
writer, (reader,) = _make_arena(n_reader=1)
|
||||
monkeypatch.setattr(shm_tensor_arena, "_ARENA_MIN_BYTES", 1 << 20)
|
||||
monkeypatch.setitem(
|
||||
shm_tensor_arena._TENSOR_ARENAS, reader.shared_memory.name, reader
|
||||
)
|
||||
nc = torch.randn(2048, 1024)[:, ::2] # non-contiguous, above threshold
|
||||
assert not nc.is_contiguous()
|
||||
data, buffers, written = _dumps_arena(nc, writer)
|
||||
out = pickle.loads(data, buffers=buffers)
|
||||
assert torch.equal(out, nc)
|
||||
# The arena never saw it: no slot consumed on the reader.
|
||||
assert written == []
|
||||
assert reader._pending_release == []
|
||||
del out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["requires_grad", "conj"])
|
||||
def test_arena_pickler_excludes_requires_grad_and_conj(monkeypatch, case: str):
|
||||
"""Tensors `_reduce_tensor` excludes (requires_grad, or an exotic
|
||||
conjugate-bit view that can't alias as uint8) must fall through the
|
||||
arena the same way, even when every other criterion qualifies."""
|
||||
writer, (reader,) = _make_arena(n_reader=1)
|
||||
monkeypatch.setattr(shm_tensor_arena, "_ARENA_MIN_BYTES", 1 << 20)
|
||||
monkeypatch.setitem(
|
||||
shm_tensor_arena._TENSOR_ARENAS, reader.shared_memory.name, reader
|
||||
)
|
||||
if case == "requires_grad":
|
||||
tensor = torch.randn(1024, 1024, requires_grad=True) # 4MiB, above threshold
|
||||
else:
|
||||
tensor = torch.randn(600, 600, dtype=torch.complex64).conj() # ~2.9MiB
|
||||
assert tensor.is_contiguous()
|
||||
assert tensor.numel() * tensor.element_size() >= shm_tensor_arena._ARENA_MIN_BYTES
|
||||
|
||||
data, buffers, written = _dumps_arena(tensor, writer)
|
||||
# Declined by the arena -- no slot consumed -- and handled by
|
||||
# _reduce_tensor's identical exclusion instead.
|
||||
assert written == []
|
||||
out = pickle.loads(data, buffers=buffers)
|
||||
assert torch.equal(out, tensor)
|
||||
assert out.requires_grad == tensor.requires_grad
|
||||
|
||||
|
||||
def test_arena_pickler_excludes_tensor_subclasses(monkeypatch):
|
||||
"""`torch.nn.Parameter` is diverted via its own `_rebuild_arena_parameter`
|
||||
hook, which wraps an intermediate tensor it keeps no Python reference to.
|
||||
The slot must stay reserved while the Parameter is alive -- the
|
||||
`_pending_release` assertions below check this even though
|
||||
`type(out) is torch.nn.Parameter` alone would not."""
|
||||
writer, (reader,) = _make_arena(n_reader=1)
|
||||
monkeypatch.setattr(shm_tensor_arena, "_ARENA_MIN_BYTES", 1 << 20)
|
||||
monkeypatch.setitem(
|
||||
shm_tensor_arena._TENSOR_ARENAS, reader.shared_memory.name, reader
|
||||
)
|
||||
# requires_grad=False to isolate the subclass check from the (already
|
||||
# separately tested) requires_grad exclusion.
|
||||
param = torch.nn.Parameter(torch.randn(1024, 1024), requires_grad=False)
|
||||
assert type(param) is not torch.Tensor
|
||||
assert param.is_contiguous()
|
||||
assert param.numel() * param.element_size() >= shm_tensor_arena._ARENA_MIN_BYTES
|
||||
|
||||
data, buffers, written = _dumps_arena(param, writer)
|
||||
# The Parameter itself is diverted directly (_ARENA_REBUILD_FNS routes
|
||||
# it to _rebuild_arena_parameter) -- Parameter.__reduce_ex__ never runs,
|
||||
# so there's no separate "inner" tensor diversion to speak of.
|
||||
assert written != []
|
||||
out = pickle.loads(data, buffers=buffers)
|
||||
# The outer Parameter's type identity survives.
|
||||
assert type(out) is torch.nn.Parameter
|
||||
assert torch.equal(out, param)
|
||||
assert out.requires_grad == param.requires_grad
|
||||
# Not yet queued for release: `out` still shares the slot's storage.
|
||||
assert reader._pending_release == []
|
||||
del out
|
||||
assert reader._pending_release == written
|
||||
_drain(reader)
|
||||
|
||||
|
||||
def test_arena_unregisters_pinned_memory_on_del(monkeypatch):
|
||||
"""A reader that cudaHostRegister-ed its mapping must cudaHostUnregister
|
||||
the same pointer before the mapping is closed; an unpinned arena must not
|
||||
unregister memory it never registered."""
|
||||
writer, (reader,) = _make_arena(n_reader=1)
|
||||
cudart = mock.MagicMock()
|
||||
cudart.cudaHostRegister.return_value = 0
|
||||
cudart.cudaHostUnregister.return_value = 0
|
||||
monkeypatch.setattr(
|
||||
shm_tensor_arena,
|
||||
"current_platform",
|
||||
SimpleNamespace(is_cuda_alike=lambda: True, cudart=lambda: cudart),
|
||||
)
|
||||
reader._ensure_pinned()
|
||||
assert reader._pinned
|
||||
ptr = cudart.cudaHostRegister.call_args.args[0]
|
||||
del reader
|
||||
cudart.cudaHostUnregister.assert_called_once_with(ptr)
|
||||
del writer # never pinned -> no unregister
|
||||
cudart.cudaHostUnregister.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
|
||||
def test_arena_event_gated_release():
|
||||
"""On the pinned path a slot release is gated on an H2D-completion CUDA
|
||||
event: the writer must not see the reader's done flag until the async DMA
|
||||
sourced from the slot has been retired. Release is first queued once the
|
||||
caller drops the CPU-side view -- the realistic pattern: the view is only
|
||||
needed as the H2D source, while the GPU tensor is what's retained for the
|
||||
rest of the step."""
|
||||
writer, (reader,) = _make_arena(n_reader=1)
|
||||
reader._ensure_pinned()
|
||||
if not reader._pinned:
|
||||
pytest.skip("cudaHostRegister unavailable in this environment")
|
||||
src = torch.randn(512, 1024)
|
||||
idx = writer.write_tensor(src)
|
||||
view = _get_view(reader, idx, src)
|
||||
dev = view.to("cuda", non_blocking=True)
|
||||
# Not yet queued: this test still holds `view`.
|
||||
assert reader._pending_release == []
|
||||
del view
|
||||
assert reader._pending_release == [idx]
|
||||
reader.flush_releases()
|
||||
# Deferred behind the event, not applied eagerly.
|
||||
assert len(reader._deferred_releases) == 1
|
||||
with reader._meta(idx) as meta:
|
||||
assert meta[1] == 0
|
||||
torch.accelerator.synchronize()
|
||||
reader.flush_releases()
|
||||
assert reader._deferred_releases == []
|
||||
with reader._meta(idx) as meta:
|
||||
assert meta[1] == 1
|
||||
assert torch.equal(dev.cpu(), src)
|
||||
del dev
|
||||
|
||||
|
||||
def test_tensor_arena_registry_is_weak_and_survives_shutdown():
|
||||
"""`_TENSOR_ARENAS` must not keep an arena alive past its owning
|
||||
MessageQueue, but also must not be cleared by `shutdown()` -- which can
|
||||
run on a different thread than an in-flight `dequeue()` still
|
||||
reconstructing a tensor from that arena. A weak-value registry gets
|
||||
both: the entry survives `shutdown()` and vanishes only once
|
||||
unreferenced."""
|
||||
writer_mq = MessageQueue(n_reader=1, n_local_reader=1, enable_shm_tensor_arena=True)
|
||||
handle = writer_mq.export_handle()
|
||||
reader_mq = MessageQueue.create_from_handle(handle, rank=0)
|
||||
|
||||
name = reader_mq.tensor_arena.shared_memory.name
|
||||
assert shm_tensor_arena._TENSOR_ARENAS[name] is reader_mq.tensor_arena
|
||||
|
||||
# shutdown() must NOT remove the entry: that would race with a
|
||||
# concurrent dequeue() still reconstructing a tensor from this arena.
|
||||
reader_mq.shutdown()
|
||||
assert shm_tensor_arena._TENSOR_ARENAS[name] is reader_mq.tensor_arena
|
||||
|
||||
# Once nothing else references the arena, the registry entry vanishes
|
||||
# on its own.
|
||||
reader_mq.tensor_arena = None
|
||||
assert name not in shm_tensor_arena._TENSOR_ARENAS
|
||||
|
||||
writer_mq.shutdown()
|
||||
|
||||
|
||||
def test_create_from_handle_tolerates_arena_attach_failure():
|
||||
"""If a reader's `ShmTensorArena.__init__` can't attach to the writer's
|
||||
segment (`FileNotFoundError`, e.g. deserialized on a different node),
|
||||
`create_from_handle` must not crash -- same deferred-failure convention
|
||||
as the sibling `ShmRingBuffer` path."""
|
||||
writer_mq = MessageQueue(n_reader=1, n_local_reader=1, enable_shm_tensor_arena=True)
|
||||
handle = writer_mq.export_handle()
|
||||
n_reader, slot_bytes, n_slots, _real_name = handle.tensor_arena_handle
|
||||
bogus_handle = dataclasses.replace(
|
||||
handle,
|
||||
tensor_arena_handle=(
|
||||
n_reader,
|
||||
slot_bytes,
|
||||
n_slots,
|
||||
"vllm-arena-does-not-exist",
|
||||
),
|
||||
)
|
||||
|
||||
reader_mq = MessageQueue.create_from_handle(bogus_handle, rank=0)
|
||||
|
||||
# Attach failed silently: the arena object exists but never got its
|
||||
# shared_memory attribute, and was never registered.
|
||||
assert reader_mq.tensor_arena is not None
|
||||
assert not hasattr(reader_mq.tensor_arena, "shared_memory")
|
||||
assert "vllm-arena-does-not-exist" not in shm_tensor_arena._TENSOR_ARENAS
|
||||
|
||||
reader_mq.shutdown()
|
||||
writer_mq.shutdown()
|
||||
|
||||
|
||||
@worker_fn_wrapper
|
||||
def worker_fn_arena_broadcast():
|
||||
rank = dist.get_rank()
|
||||
writer_rank = 0
|
||||
if rank == writer_rank:
|
||||
message_queue = MessageQueue(
|
||||
1,
|
||||
1,
|
||||
local_reader_ranks=[1],
|
||||
max_chunk_bytes=8 * 1024 * 1024,
|
||||
enable_shm_tensor_arena=True,
|
||||
)
|
||||
handles = [message_queue.export_handle()]
|
||||
dist.broadcast_object_list(handles, src=writer_rank)
|
||||
else:
|
||||
handles = [None]
|
||||
dist.broadcast_object_list(handles, src=writer_rank)
|
||||
message_queue = MessageQueue.create_from_handle(handles[0], rank)
|
||||
message_queue.wait_until_ready()
|
||||
|
||||
torch.manual_seed(42)
|
||||
payload = {
|
||||
# 16MiB: above the arena divert threshold (8MiB) -> arena slot.
|
||||
"huge": torch.randn(2048, 2048),
|
||||
# 2MiB: declined by the arena -> out-of-band _reduce_tensor path.
|
||||
"mid": torch.randn(1024, 512),
|
||||
}
|
||||
|
||||
if rank == writer_rank:
|
||||
assert message_queue.tensor_arena is not None
|
||||
with mock.patch(
|
||||
"vllm.distributed.device_communicators.shm_broadcast._reduce_tensor",
|
||||
wraps=_reduce_tensor,
|
||||
) as wrapped_reduce:
|
||||
message_queue.enqueue(payload)
|
||||
# "huge" was diverted into the arena before the dispatch table was
|
||||
# consulted; only "mid" went through _reduce_tensor.
|
||||
assert wrapped_reduce.call_count == 1
|
||||
else:
|
||||
received = message_queue.dequeue(timeout=30)
|
||||
assert torch.equal(received["huge"], payload["huge"])
|
||||
assert torch.equal(received["mid"], payload["mid"])
|
||||
# The huge tensor is a zero-copy view into the arena's shared memory
|
||||
# (not a copy received over the transport). Its slot is not yet
|
||||
# queued for release while `received` still holds the reference.
|
||||
arena = message_queue.tensor_arena
|
||||
assert arena is not None
|
||||
assert arena._pending_release == []
|
||||
huge_ptr = received["huge"].data_ptr()
|
||||
arena_base = torch.frombuffer(
|
||||
arena.shared_memory.buf, dtype=torch.uint8
|
||||
).data_ptr()
|
||||
assert arena_base <= huge_ptr < arena_base + arena.total_bytes
|
||||
# Dropping the last reference to the slot's storage queues it for
|
||||
# release (see get_tensor).
|
||||
del received
|
||||
assert len(arena._pending_release) == 1
|
||||
|
||||
dist.barrier()
|
||||
print(f"arena broadcast passed the test! Rank {rank}")
|
||||
|
||||
|
||||
def test_arena_broadcast():
|
||||
# Spawn (not fork): by the time this test runs, the pytest process may be
|
||||
# multi-threaded / CUDA-initialized (earlier GPU tests), and a forked
|
||||
# child running the arena's slot memcpy can deadlock on inherited lock
|
||||
# state. Spawned workers start clean; the larger timeout absorbs their
|
||||
# interpreter + import startup.
|
||||
distributed_run(
|
||||
worker_fn_arena_broadcast,
|
||||
2,
|
||||
timeout=180,
|
||||
mp_context=mp.get_context("spawn"),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user