[Core] shm tensor arena for efficient cpu->gpu worker broadcast (#51207)
Signed-off-by: Bolin Sun <bolins@nvidia.com> Signed-off-by: <bolins@nvidia.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude
Nick Hill
parent
fa94ad4143
commit
04b02fc8a5
@@ -235,6 +235,15 @@ class ParallelConfig:
|
||||
disable_custom_all_reduce: bool = False
|
||||
"""Disable the custom all-reduce kernel and fall back to NCCL."""
|
||||
|
||||
enable_shm_tensor_arena: bool = False
|
||||
"""Route large CPU tensors (e.g. multimodal ``pixel_values``) in the
|
||||
engine→worker broadcast through a zero-copy shared-memory arena instead of
|
||||
sending a copy to every reader. **Experimental**: may be reconciled with
|
||||
vLLM's existing multimodal shm tensor caching in a future release. Opt-in;
|
||||
most beneficial for multimodal serving with large images and tensor
|
||||
parallelism. Active only when all queue readers are node-local; reserves
|
||||
slots in ``/dev/shm``."""
|
||||
|
||||
enable_elastic_ep: bool = False
|
||||
"""Enable elastic expert parallelism with stateless NCCL groups for DP/EP."""
|
||||
elastic_ep_max_dp_size: int = Field(default=None, ge=1) # type: ignore[assignment]
|
||||
@@ -901,6 +910,7 @@ class ParallelConfig:
|
||||
"nnodes",
|
||||
"max_parallel_loading_workers",
|
||||
"disable_custom_all_reduce",
|
||||
"enable_shm_tensor_arena",
|
||||
"ray_workers_use_nsight",
|
||||
"ray_runtime_env",
|
||||
"placement_group",
|
||||
|
||||
@@ -61,6 +61,10 @@ SPINLOOP_TIMEOUT_SECONDS = 0.1
|
||||
if TYPE_CHECKING:
|
||||
from _typeshed import SizedBuffer
|
||||
|
||||
from vllm.distributed.device_communicators.shm_tensor_arena import (
|
||||
ShmTensorArena,
|
||||
)
|
||||
|
||||
VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
|
||||
# Cap on how long an idle reader parks before re-reading the authoritative SHM
|
||||
# written-flag. Bounds lost-notify recovery latency to ~5s while the periodic
|
||||
@@ -471,6 +475,7 @@ class Handle:
|
||||
local_notify_addr: str | None = None
|
||||
remote_subscribe_addr: str | None = None
|
||||
remote_addr_ipv6: bool = False
|
||||
tensor_arena_handle: tuple[int, int, int, str] | None = None
|
||||
|
||||
|
||||
def _close_zmq(context: zmq.Context, sockets: list[zmq.Socket]) -> None:
|
||||
@@ -490,6 +495,7 @@ class MessageQueue:
|
||||
max_chunk_bytes: int = 1024 * 1024 * 24,
|
||||
max_chunks: int = 10,
|
||||
connect_ip: str | None = None,
|
||||
enable_shm_tensor_arena: bool = False,
|
||||
):
|
||||
if local_reader_ranks is None:
|
||||
local_reader_ranks = list(range(n_local_reader))
|
||||
@@ -501,12 +507,22 @@ class MessageQueue:
|
||||
self.shutting_down = False
|
||||
context = Context()
|
||||
|
||||
self.tensor_arena: ShmTensorArena | None = None
|
||||
if n_local_reader > 0:
|
||||
# for local readers, we will:
|
||||
# 1. create a shared memory ring buffer to communicate small data
|
||||
# 2. create a publish-subscribe socket to communicate large data
|
||||
self.buffer = ShmRingBuffer(n_local_reader, max_chunk_bytes, max_chunks)
|
||||
|
||||
# Zero-copy arena for large CPU tensors (see ShmTensorArena).
|
||||
# Local readers only: remote readers receive the pickled bytes
|
||||
# over a socket and cannot map the arena, so the substitution
|
||||
# would break them.
|
||||
if enable_shm_tensor_arena and n_remote_reader == 0:
|
||||
from vllm.distributed.device_communicators import shm_tensor_arena
|
||||
|
||||
self.tensor_arena = shm_tensor_arena.ShmTensorArena(n_local_reader)
|
||||
|
||||
# XPUB is very similar to PUB,
|
||||
# except that it can receive subscription messages
|
||||
# to confirm the number of subscribers
|
||||
@@ -567,6 +583,9 @@ class MessageQueue:
|
||||
local_notify_addr=local_notify_addr,
|
||||
remote_subscribe_addr=remote_subscribe_addr,
|
||||
remote_addr_ipv6=remote_addr_ipv6,
|
||||
tensor_arena_handle=(
|
||||
self.tensor_arena.handle() if self.tensor_arena is not None else None
|
||||
),
|
||||
)
|
||||
self._close_zmq_on_collect(context)
|
||||
|
||||
@@ -599,6 +618,7 @@ class MessageQueue:
|
||||
|
||||
context = Context()
|
||||
|
||||
self.tensor_arena = None
|
||||
if rank in handle.local_reader_ranks:
|
||||
assert handle.buffer_handle is not None
|
||||
self.buffer = ShmRingBuffer(*handle.buffer_handle)
|
||||
@@ -607,6 +627,23 @@ class MessageQueue:
|
||||
self._is_local_reader = True
|
||||
self._is_remote_reader = False
|
||||
|
||||
arena_handle = handle.tensor_arena_handle
|
||||
if arena_handle is not None:
|
||||
from vllm.distributed.device_communicators import shm_tensor_arena
|
||||
|
||||
self.tensor_arena = shm_tensor_arena.ShmTensorArena(
|
||||
*arena_handle, reader_rank=self.local_reader_rank
|
||||
)
|
||||
# `shared_memory` is unset (not just None) if attaching to
|
||||
# the writer's segment raised FileNotFoundError -- see
|
||||
# ShmTensorArena.__init__'s reader branch. Registering a
|
||||
# broken arena is a no-op deferred failure (same convention
|
||||
# ShmRingBuffer uses above), not a crash here.
|
||||
if hasattr(self.tensor_arena, "shared_memory"):
|
||||
shm_tensor_arena._TENSOR_ARENAS[
|
||||
self.tensor_arena.shared_memory.name
|
||||
] = self.tensor_arena
|
||||
|
||||
self.local_socket = context.socket(SUB)
|
||||
self.local_socket.setsockopt_string(SUBSCRIBE, "")
|
||||
socket_addr = handle.local_subscribe_addr
|
||||
@@ -880,12 +917,21 @@ class MessageQueue:
|
||||
# dispatch table would otherwise shadow them.
|
||||
dispatch_table = dict(copyreg.dispatch_table)
|
||||
dispatch_table[torch.Tensor] = _reduce_tensor
|
||||
arena = self.tensor_arena
|
||||
with io.BytesIO() as bio:
|
||||
pickler = pickle.Pickler(
|
||||
bio,
|
||||
protocol=pickle.HIGHEST_PROTOCOL,
|
||||
buffer_callback=oob_callback,
|
||||
)
|
||||
pickler: pickle.Pickler
|
||||
if arena is not None:
|
||||
from vllm.distributed.device_communicators.shm_tensor_arena import (
|
||||
_ArenaPickler,
|
||||
)
|
||||
|
||||
pickler = _ArenaPickler(bio, arena, buffer_callback=oob_callback)
|
||||
else:
|
||||
pickler = pickle.Pickler(
|
||||
bio,
|
||||
protocol=pickle.HIGHEST_PROTOCOL,
|
||||
buffer_callback=oob_callback,
|
||||
)
|
||||
pickler.dispatch_table = dispatch_table
|
||||
pickler.dump(obj)
|
||||
all_buffers[0] = bio.getvalue()
|
||||
@@ -922,6 +968,9 @@ class MessageQueue:
|
||||
):
|
||||
"""Read from message queue with optional timeout (in seconds)"""
|
||||
if self._is_local_reader:
|
||||
# See ShmTensorArena.flush_releases for what this retires and why.
|
||||
if self.tensor_arena is not None:
|
||||
self.tensor_arena.flush_releases()
|
||||
with self.acquire_read(timeout, indefinite) as buf:
|
||||
overflow = buf[0] == 1
|
||||
if not overflow:
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Zero-copy shared-memory arena for large CPU tensors (multimodal fast path).
|
||||
|
||||
Problem: `MessageQueue.enqueue` (see `shm_broadcast.py`) pickles the whole
|
||||
payload, and torch's default tensor pickling copies bytes into the pickle
|
||||
stream on the writer and back out on every local reader. For a large
|
||||
`pixel_values` tensor (up to hundreds of MB) this blocks the EngineCore step
|
||||
loop for ~1s with all GPUs idle.
|
||||
|
||||
Fast path: `_ArenaPickler.reducer_override` intercepts qualifying CPU
|
||||
tensors during pickling, memcpys them ONCE into a free arena slot, and
|
||||
pickles a tiny `(arena_name, slot, nbytes, dtype, shape)` stub. Readers
|
||||
rebuild a zero-copy view of the mapped slot (`torch.frombuffer`).
|
||||
|
||||
Slot lifecycle: single writer, n_reader readers, per-slot metadata
|
||||
[written_flag, reader0_done, ..., readerN_done] (same protocol as
|
||||
ShmRingBuffer). A slot is not released when rebuilt: the rebuilt tensor is
|
||||
the SOURCE of an async H2D copy, and callers like chunked-prefill
|
||||
`prompt_embeds` retain it across many steps. Release is queued only once the
|
||||
tensor's storage is freed, i.e. once no tensor or view of the slot remains
|
||||
(see `get_tensor`), and is further gated on a CUDA event on the pinned fast
|
||||
path, since the H2D can outlive the step that issued it (see
|
||||
`flush_releases`).
|
||||
|
||||
The writer NEVER blocks: no free slot (or an oversized tensor) falls back to
|
||||
the default pickle path, so worst case matches today's behavior.
|
||||
|
||||
Physical memory: slots are paged in lazily, so an arena on a queue that
|
||||
never carries big tensors costs ~0. Creation is guarded by
|
||||
`check_shm_free_space` (like `ShmRingBuffer`) so an undersized `/dev/shm`
|
||||
fails fast instead of a `SIGBUS` mid-copy.
|
||||
"""
|
||||
|
||||
import pickle
|
||||
import weakref
|
||||
from contextlib import suppress
|
||||
from multiprocessing import shared_memory
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
check_shm_free_space,
|
||||
memory_fence,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Arena sizing — internal constants (kept off the env/CLI surface).
|
||||
_ARENA_SLOTS = 8
|
||||
_ARENA_SLOT_BYTES = 256 * 1024 * 1024
|
||||
# Minimum contiguous CPU-tensor size to divert into the arena; smaller tensors
|
||||
# take the out-of-band pickle-buffer path.
|
||||
_ARENA_MIN_BYTES = 8 * 1024 * 1024
|
||||
|
||||
# Reader-side registry: arena shm name -> attached ShmTensorArena, keyed by
|
||||
# name rather than a single slot since a process isn't guaranteed to attach
|
||||
# to only one. Populated by `MessageQueue.create_from_handle`; consumed by
|
||||
# the `_rebuild_arena_*` unpickle hooks. WEAK-value: a strong reference here
|
||||
# would keep the arena (and its pinned mapping) alive for the process
|
||||
# lifetime, and would need `MessageQueue.shutdown` to remove it explicitly --
|
||||
# unsafe since shutdown() can run on a different thread than an in-flight
|
||||
# `dequeue`. Instead an entry just disappears once nothing else references
|
||||
# that arena.
|
||||
_TENSOR_ARENAS: weakref.WeakValueDictionary[str, "ShmTensorArena"] = (
|
||||
weakref.WeakValueDictionary()
|
||||
)
|
||||
|
||||
|
||||
class ShmTensorArena:
|
||||
"""Slotted shared-memory arena: one writer, n_reader zero-copy readers.
|
||||
|
||||
Memory layout: [slot0 | slot1 | ... | slotN-1 | meta0 | meta1 | ... ]
|
||||
where each meta is (1 + n_reader) bytes: [written_flag, reader_done...].
|
||||
Slot states mirror ShmRingBuffer:
|
||||
written=0 -> free (never written / being written)
|
||||
written=1, some reader_done=0 -> in use, cannot reuse
|
||||
written=1, all reader_done=1 -> consumed, can reuse
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
n_reader: int,
|
||||
slot_bytes: int = _ARENA_SLOT_BYTES,
|
||||
n_slots: int = _ARENA_SLOTS,
|
||||
name: str | None = None,
|
||||
reader_rank: int = -1,
|
||||
):
|
||||
self.n_reader = n_reader
|
||||
self.slot_bytes = slot_bytes
|
||||
self.n_slots = n_slots
|
||||
self.reader_rank = reader_rank # -1 for the writer
|
||||
self.metadata_size = 1 + n_reader
|
||||
self.metadata_offset = slot_bytes * n_slots
|
||||
self.total_bytes = (slot_bytes + self.metadata_size) * n_slots
|
||||
self._next_slot = 0
|
||||
self._fallbacks = 0
|
||||
self._pin_attempted = False
|
||||
self._pinned = False
|
||||
self._pinned_ptr = 0
|
||||
# slots queued for release once the storage of the tensor `get_tensor`
|
||||
# handed out is freed; flushed at the next dequeue.
|
||||
self._pending_release: list[int] = []
|
||||
# slots whose release is gated on an H2D-completion CUDA event: each
|
||||
# entry is (event, [slot_idx, ...]). Only populated on the pinned fast
|
||||
# path, where the source-side DMA is asynchronous (see flush_releases).
|
||||
self._deferred_releases: list[tuple[torch.cuda.Event, list[int]]] = []
|
||||
|
||||
if name is None:
|
||||
self.is_creator = True
|
||||
# Same guard as ShmRingBuffer: SharedMemory/ftruncate can succeed
|
||||
# on an undersized /dev/shm, so check up front instead of SIGBUS
|
||||
# on first write past tmpfs capacity.
|
||||
check_shm_free_space(self.total_bytes, allocation_name="ShmTensorArena")
|
||||
self.shared_memory = shared_memory.SharedMemory(
|
||||
create=True, size=self.total_bytes
|
||||
)
|
||||
assert self.shared_memory.buf is not None
|
||||
with self.shared_memory.buf[self.metadata_offset :] as meta:
|
||||
torch.frombuffer(meta, dtype=torch.uint8).fill_(0)
|
||||
else:
|
||||
self.is_creator = False
|
||||
# same resource_tracker workaround as ShmRingBuffer
|
||||
with patch(
|
||||
"multiprocessing.resource_tracker.register",
|
||||
lambda *args, **kwargs: None,
|
||||
):
|
||||
try:
|
||||
self.shared_memory = shared_memory.SharedMemory(name=name)
|
||||
assert self.shared_memory.size >= self.total_bytes
|
||||
except FileNotFoundError:
|
||||
# deserialized on a different node; arena unused there
|
||||
pass
|
||||
|
||||
def handle(self):
|
||||
return (self.n_reader, self.slot_bytes, self.n_slots, self.shared_memory.name)
|
||||
|
||||
def _meta(self, idx: int) -> memoryview:
|
||||
start = self.metadata_offset + idx * self.metadata_size
|
||||
assert self.shared_memory.buf is not None
|
||||
return self.shared_memory.buf[start : start + self.metadata_size]
|
||||
|
||||
def _slot(self, idx: int, nbytes: int) -> memoryview:
|
||||
start = idx * self.slot_bytes
|
||||
assert self.shared_memory.buf is not None
|
||||
return self.shared_memory.buf[start : start + nbytes]
|
||||
|
||||
# ---- writer side ----
|
||||
|
||||
def write_tensor(self, t: torch.Tensor) -> int | None:
|
||||
"""Copy tensor bytes into a free slot; return slot idx or None
|
||||
(caller must then fall back to the default pickle path)."""
|
||||
nbytes = t.numel() * t.element_size()
|
||||
if nbytes > self.slot_bytes:
|
||||
return None
|
||||
try:
|
||||
# Computed before claiming a slot: exotic tensors (e.g. conjugate
|
||||
# bit set) raise RuntimeError here, same as `_reduce_tensor`'s
|
||||
# identical view -- fail before claiming, not after.
|
||||
src = t.detach().reshape(-1).view(torch.uint8)
|
||||
except RuntimeError:
|
||||
return None
|
||||
memory_fence()
|
||||
for probe in range(self.n_slots):
|
||||
idx = (self._next_slot + probe) % self.n_slots
|
||||
with self._meta(idx) as meta:
|
||||
free = meta[0] == 0 or sum(meta[1:]) == self.n_reader
|
||||
if not free:
|
||||
continue
|
||||
meta[0] = 0 # claim
|
||||
slot_mv = self._slot(idx, nbytes)
|
||||
try:
|
||||
dst = torch.frombuffer(slot_mv, dtype=torch.uint8, count=nbytes)
|
||||
dst.copy_(src)
|
||||
finally:
|
||||
slot_mv.release()
|
||||
with self._meta(idx) as meta:
|
||||
for i in range(1, self.n_reader + 1):
|
||||
meta[i] = 0
|
||||
memory_fence()
|
||||
meta[0] = 1
|
||||
memory_fence()
|
||||
self._next_slot = (idx + 1) % self.n_slots
|
||||
return idx
|
||||
self._fallbacks += 1
|
||||
if self._fallbacks == 1 or self._fallbacks % 100 == 0:
|
||||
logger.info(
|
||||
"ShmTensorArena: no free slot (%d bytes, %d fallbacks so far); "
|
||||
"falling back to the out-of-band pickle path.",
|
||||
nbytes,
|
||||
self._fallbacks,
|
||||
)
|
||||
return None
|
||||
|
||||
# ---- reader side ----
|
||||
|
||||
def _ensure_pinned(self):
|
||||
"""Pin the whole arena mapping via cudaHostRegister in THIS process
|
||||
(lazy, once). Without it the HtoD of a zero-copy tensor pays
|
||||
first-touch page faults on the tmpfs mapping plus pageable staging
|
||||
(~hundreds of ms for 200MB); registration allocates+pins the pages
|
||||
once, making every later HtoD a true DMA. Failure (no CUDA in this
|
||||
process, etc.) is fine — the copy still works, just slower."""
|
||||
if self._pin_attempted:
|
||||
return
|
||||
self._pin_attempted = True
|
||||
try:
|
||||
if not current_platform.is_cuda_alike():
|
||||
return
|
||||
import ctypes
|
||||
|
||||
buf = self.shared_memory.buf
|
||||
assert buf is not None
|
||||
ptr = ctypes.addressof(ctypes.c_char.from_buffer(buf))
|
||||
ret = current_platform.cudart().cudaHostRegister(ptr, self.total_bytes, 0)
|
||||
self._pinned = int(ret) == 0
|
||||
if self._pinned:
|
||||
self._pinned_ptr = ptr
|
||||
logger.info(
|
||||
"ShmTensorArena: cudaHostRegister(%d MB) -> %s",
|
||||
self.total_bytes >> 20,
|
||||
"pinned" if self._pinned else f"error {int(ret)}",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.info("ShmTensorArena: host-register skipped: %s", e)
|
||||
|
||||
def _unpin(self):
|
||||
"""Undo the pin from _ensure_pinned via cudaHostUnregister; must run
|
||||
before the mapping is closed. Failures are ignored — at interpreter
|
||||
shutdown the CUDA context may already be gone, and the registration
|
||||
dies with the process anyway."""
|
||||
if not self._pinned:
|
||||
return
|
||||
self._pinned = False
|
||||
with suppress(Exception):
|
||||
current_platform.cudart().cudaHostUnregister(self._pinned_ptr)
|
||||
|
||||
def get_tensor(
|
||||
self, idx: int, nbytes: int, dtype: torch.dtype, shape: tuple[int, ...]
|
||||
) -> torch.Tensor:
|
||||
"""Zero-copy view of a slot as a tensor. The slot is queued for
|
||||
release once the returned tensor's storage is freed, i.e. once the
|
||||
caller has dropped it along with any views/slices or wrappers (such
|
||||
as `torch.nn.Parameter`) sharing that storage."""
|
||||
self._ensure_pinned()
|
||||
# The tensor's storage holds the only reference to `holder` and drops
|
||||
# it when freed, firing the finalizer. (`weakref.finalize` can't
|
||||
# target the tensor itself, which views and wrappers don't keep alive,
|
||||
# nor the memoryview, which doesn't support weakrefs.)
|
||||
holder = np.frombuffer(self._slot(idx, nbytes), dtype=np.uint8)
|
||||
# Binds to the current list OBJECT: `flush_releases` must mutate it in
|
||||
# place, never rebind `self._pending_release`.
|
||||
weakref.finalize(holder, self._pending_release.append, idx)
|
||||
return torch.from_numpy(holder).view(dtype).view(shape)
|
||||
|
||||
def _mark_released(self, idxs: list[int]):
|
||||
"""Set THIS reader's done flag on the given slots; a slot becomes
|
||||
reusable once every reader has done so."""
|
||||
for idx in idxs:
|
||||
with self._meta(idx) as meta:
|
||||
meta[1 + self.reader_rank] = 1
|
||||
memory_fence()
|
||||
|
||||
def _record_release_event(self):
|
||||
"""Record a CUDA event on the current (compute) stream, or return None
|
||||
if this process has no usable CUDA context. Called from flush_releases,
|
||||
which runs after the previous step's H2D was enqueued on that same
|
||||
stream, so the event is ordered strictly after that H2D."""
|
||||
try:
|
||||
event = current_platform.Event()
|
||||
event.record()
|
||||
return event
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def flush_releases(self):
|
||||
"""Retire this reader's arena slots that are ready for writer reuse.
|
||||
|
||||
Called on every `MessageQueue.dequeue`. `_pending_release` holds
|
||||
slots whose tensor storage was just freed (see `get_tensor`). On
|
||||
the pinned fast path that tensor was the source of an async
|
||||
`non_blocking=True` H2D that can outlive the collection, so instead
|
||||
of marking released immediately we record a CUDA event (ordered
|
||||
after that H2D on the same stream) and move the slot to
|
||||
`_deferred_releases` until `event.query()` says it's done -- a
|
||||
not-yet-complete slot just waits one more `dequeue`. Unpinned (or no
|
||||
CUDA context), the H2D already staged synchronously, so the slot is
|
||||
safe to release right away.
|
||||
|
||||
A slot only becomes free for the writer (`write_tensor`) once every
|
||||
reader has marked it released (`_mark_released`).
|
||||
"""
|
||||
if self._deferred_releases:
|
||||
still_pending = []
|
||||
for deferred_event, idxs in self._deferred_releases:
|
||||
if deferred_event.query():
|
||||
self._mark_released(idxs)
|
||||
else:
|
||||
still_pending.append((deferred_event, idxs))
|
||||
self._deferred_releases = still_pending
|
||||
|
||||
if not self._pending_release:
|
||||
return
|
||||
# Drain in place with atomic pops, never rebind (see get_tensor): a
|
||||
# finalizer may append from another thread concurrently.
|
||||
idxs = []
|
||||
while self._pending_release:
|
||||
idxs.append(self._pending_release.pop())
|
||||
|
||||
event = self._record_release_event() if self._pinned else None
|
||||
if event is None:
|
||||
self._mark_released(idxs)
|
||||
else:
|
||||
self._deferred_releases.append((event, idxs))
|
||||
|
||||
def __del__(self):
|
||||
if hasattr(self, "shared_memory"):
|
||||
self._unpin()
|
||||
try:
|
||||
self.shared_memory.close()
|
||||
if self.is_creator:
|
||||
self.shared_memory.unlink()
|
||||
except BufferError:
|
||||
# zero-copy tensor views may still hold exported pointers at
|
||||
# interpreter shutdown; the mapping dies with the process.
|
||||
pass
|
||||
|
||||
|
||||
def _rebuild_arena_tensor(arena_name, slot_idx, nbytes, dtype_str, shape):
|
||||
"""Unpickle hook: rebuild a tensor as a zero-copy view of an arena slot."""
|
||||
arena = _TENSOR_ARENAS[arena_name]
|
||||
return arena.get_tensor(slot_idx, nbytes, getattr(torch, dtype_str), shape)
|
||||
|
||||
|
||||
def _rebuild_arena_parameter(arena_name, slot_idx, nbytes, dtype_str, shape):
|
||||
"""Unpickle hook for `torch.nn.Parameter` (shares the slot's storage)."""
|
||||
t = _rebuild_arena_tensor(arena_name, slot_idx, nbytes, dtype_str, shape)
|
||||
return torch.nn.Parameter(t, requires_grad=False)
|
||||
|
||||
|
||||
# Types `_ArenaPickler.reducer_override` diverts, mapped to the rebuild hook
|
||||
# that reconstructs them. Any other type -- including other `torch.Tensor`
|
||||
# subclasses -- falls through to the normal pickle path, since an arbitrary
|
||||
# subclass can't be faithfully rebuilt from just the slot's bytes.
|
||||
_ARENA_REBUILD_FNS = {
|
||||
torch.Tensor: _rebuild_arena_tensor,
|
||||
torch.nn.Parameter: _rebuild_arena_parameter,
|
||||
}
|
||||
|
||||
|
||||
class _ArenaPickler(pickle.Pickler):
|
||||
"""Pickler that diverts large contiguous CPU tensors into the arena.
|
||||
|
||||
`reducer_override` is consulted before an object's normal reduction. For
|
||||
a qualifying tensor (exact type in `_ARENA_REBUILD_FNS`, CPU, strided,
|
||||
contiguous, not `requires_grad`, above the divert-size threshold -- the
|
||||
same criteria `_reduce_tensor` uses, plus size) it does one memcpy into a
|
||||
free slot and returns a tiny rebuild stub. Anything that doesn't qualify,
|
||||
or finds the arena full, returns `NotImplemented` and falls through to
|
||||
the normal out-of-band pickle path, exactly as if no arena were attached.
|
||||
"""
|
||||
|
||||
def __init__(self, file, arena: ShmTensorArena, buffer_callback=None):
|
||||
super().__init__(
|
||||
file,
|
||||
protocol=pickle.HIGHEST_PROTOCOL,
|
||||
buffer_callback=buffer_callback,
|
||||
)
|
||||
self.arena = arena
|
||||
|
||||
def reducer_override(self, obj):
|
||||
rebuild_fn = _ARENA_REBUILD_FNS.get(type(obj))
|
||||
if (
|
||||
rebuild_fn is not None
|
||||
and obj.device.type == "cpu"
|
||||
and obj.layout == torch.strided
|
||||
and obj.is_contiguous()
|
||||
and not obj.requires_grad
|
||||
and obj.numel() * obj.element_size() >= _ARENA_MIN_BYTES
|
||||
):
|
||||
idx = self.arena.write_tensor(obj)
|
||||
if idx is not None:
|
||||
return (
|
||||
rebuild_fn,
|
||||
(
|
||||
self.arena.shared_memory.name,
|
||||
idx,
|
||||
obj.numel() * obj.element_size(),
|
||||
str(obj.dtype).removeprefix("torch."),
|
||||
tuple(obj.shape),
|
||||
),
|
||||
)
|
||||
return NotImplemented
|
||||
@@ -622,6 +622,7 @@ class EngineArgs:
|
||||
allow_deprecated_quantization: bool = ModelConfig.allow_deprecated_quantization
|
||||
enforce_eager: bool = ModelConfig.enforce_eager
|
||||
disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
|
||||
enable_shm_tensor_arena: bool = ParallelConfig.enable_shm_tensor_arena
|
||||
language_model_only: bool = MultiModalConfig.language_model_only
|
||||
limit_mm_per_prompt: dict[str, int | dict[str, int]] = get_field(
|
||||
MultiModalConfig, "limit_per_prompt"
|
||||
@@ -1274,6 +1275,10 @@ class EngineArgs:
|
||||
"--all2all-backend", **parallel_kwargs["all2all_backend"]
|
||||
)
|
||||
parallel_group.add_argument("--enable-dbo", **parallel_kwargs["enable_dbo"])
|
||||
parallel_group.add_argument(
|
||||
"--enable-shm-tensor-arena",
|
||||
**parallel_kwargs["enable_shm_tensor_arena"],
|
||||
)
|
||||
parallel_group.add_argument(
|
||||
"--ubatch-size",
|
||||
**parallel_kwargs["ubatch_size"],
|
||||
@@ -2534,6 +2539,7 @@ class EngineArgs:
|
||||
enable_elastic_ep=self.enable_elastic_ep,
|
||||
elastic_ep_max_dp_size=self.elastic_ep_max_dp_size,
|
||||
enable_dbo=self.enable_dbo,
|
||||
enable_shm_tensor_arena=self.enable_shm_tensor_arena,
|
||||
ubatch_size=self.ubatch_size,
|
||||
dbo_decode_token_threshold=self.dbo_decode_token_threshold,
|
||||
dp_sync_interval=self.dp_sync_interval,
|
||||
|
||||
@@ -166,6 +166,7 @@ class MultiprocExecutor(Executor):
|
||||
self.local_world_size,
|
||||
max_chunk_bytes=max_chunk_bytes,
|
||||
connect_ip=mq_connect_ip,
|
||||
enable_shm_tensor_arena=(self.parallel_config.enable_shm_tensor_arena),
|
||||
)
|
||||
scheduler_output_handle = self.rpc_broadcast_mq.export_handle()
|
||||
# Create workers
|
||||
|
||||
Reference in New Issue
Block a user