# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import contextlib import ctypes import errno import functools import mmap import os import time from collections.abc import Callable, Iterator from ctypes import c_int, c_long, c_size_t, c_void_p import numpy as np import torch from vllm.distributed.device_communicators.shm_broadcast import ( check_shm_free_space, ) from vllm.logger import init_logger from vllm.utils.math_utils import cdiv, round_up from vllm.v1.kv_offload.cpu.host_register import host_unregister logger = init_logger(__name__) # MADV_POPULATE_WRITE was added in Linux 5.14 (value 23). _MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23) _MAP_FIXED = 0x10 # Page-cache inserts are serialized per shm file, capping pre-faulting of one # file near 10 GiB/s; smaller segments are not worth a file. _MIN_SEGMENT_BYTES = 1 << 30 def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0) -> None: """Spin-wait until the file reaches expected_size (creator truncated it).""" deadline = time.monotonic() + timeout while True: if os.fstat(fd).st_size >= expected_size: return if os.fstat(fd).st_nlink == 0: raise RuntimeError( "Shared offload region creator failed during initialization." ) if time.monotonic() > deadline: raise TimeoutError( f"Timed out waiting for mmap file to reach {expected_size} bytes" ) time.sleep(0.005) @functools.cache def _libc_mmap() -> Callable[..., int | None]: mmap_fn = ctypes.CDLL(None, use_errno=True).mmap mmap_fn.argtypes = [c_void_p, c_size_t, c_int, c_int, c_int, c_long] mmap_fn.restype = c_void_p return mmap_fn def _madvise_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None: mmap_obj.madvise(_MADV_POPULATE_WRITE, offset, length) def _fallback_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None: # Touch one byte per page via a read-modify-write so existing bytes are # preserved — a peer worker may have already written KV data into this # shared mmap by the time we run on a kernel without MADV_POPULATE_WRITE. arr = np.frombuffer(mmap_obj, dtype=np.uint8) arr[offset : offset + length : mmap.PAGESIZE] |= 0 def _get_populate_write_fn( mmap_obj: mmap.mmap, ) -> Callable[[mmap.mmap, int, int], None]: """Select the pre-faulting method once for this mmap.""" try: _madvise_populate_write(mmap_obj, 0, mmap.PAGESIZE) except OSError as e: if e.errno != errno.EINVAL: raise logger.warning( "MADV_POPULATE_WRITE is not supported; falling back to per-page " "writes for mmap pre-population. Startup may be slower." ) return _fallback_populate_write return _madvise_populate_write class SharedOffloadRegion: """Single mmap-backed memory region shared across all workers for a vLLM instance. Workers coordinate via the filesystem: the first worker to open the file with O_EXCL becomes the creator and calls ftruncate; the rest open the existing file and wait until it reaches the expected size. Each worker then mmap()s the full file. File path: /dev/shm/vllm_offload_{engine_id}.mmap. When a barrier is given, the path is unlinked once every worker has mapped the file, so the kernel reclaims the memory when the last worker exits, no matter how it exits; mappings taken before the unlink stay valid. ``populate_shard=(index, count)`` pre-faults only this worker's contiguous share, for layouts where every worker maps the same bytes. The barrier also waits for all shards to finish pre-faulting before returning. ``num_segments`` splits backing files beneath the contiguous mapping. """ BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE def __init__( self, engine_id: str, num_chunks: int, rank: int | None, kv_bytes_per_chunk: int, cpu_page_size: int, barrier: Callable[[], None] | None = None, *, creator_memory_check: Callable[[int], None] | None = None, populate_shard: tuple[int, int] | None = None, num_segments: int = 1, ) -> None: self.page_size = mmap.PAGESIZE assert kv_bytes_per_chunk % self.page_size == 0 self.num_chunks = num_chunks self._row_stride = kv_bytes_per_chunk self.total_size_bytes = self.num_chunks * self._row_stride self.cpu_page_size = cpu_page_size self.mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap" num_segments = min(num_segments, self.total_size_bytes // _MIN_SEGMENT_BYTES) rows = max(1, cdiv(num_chunks, max(1, num_segments))) # (path, offset, length) per file. self._segments = [ ( f"{self.mmap_path}.{i}" if i else self.mmap_path, row * self._row_stride, min(rows, num_chunks - row) * self._row_stride, ) for i, row in enumerate(range(0, max(num_chunks, 1), rows)) ] self._creator = False # set True only if this worker creates the file self.rank = rank if rank is not None: # byte offset to this worker's first slot within each chunk row self._worker_offset = rank * cpu_page_size # exclusive upper bound for this worker's area within each row self._worker_area_end = (rank + 1) * cpu_page_size barriers_remaining = 2 if populate_shard is not None else 1 try: try: self.fd: int | None = os.open( self.mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600 ) except FileExistsError: # Joiner path — another worker won O_EXCL. Reopen and wait # for the file to reach expected size. self.fd = os.open(self.mmap_path, os.O_RDWR) _wait_for_file_size(self.fd, self._segments[0][2]) logger.info("Opened existing mmap file %s", self.mmap_path) else: # Creator path. We won O_EXCL, so we own the file: any # failure here must clean up so concurrent joiners don't # land on a 0-byte stub and spin in _wait_for_file_size # for the full 30 s timeout. self._creator = True if creator_memory_check is not None: creator_memory_check(self.total_size_bytes) check_shm_free_space( self.total_size_bytes, allocation_name="CPU KV offload shared region in /dev/shm", ) for path, _, length in self._segments[1:]: fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600) os.ftruncate(fd, length) os.close(fd) # Joiners wait for the first file, so size it last. os.ftruncate(self.fd, self._segments[0][2]) logger.info( "Created mmap file %s (%.2f GB)", self.mmap_path, self.total_size_bytes / 1e9, ) single = len(self._segments) == 1 self.mmap_obj: mmap.mmap | None = mmap.mmap( self.fd if single else -1, self.total_size_bytes, flags=mmap.MAP_SHARED if single else mmap.MAP_PRIVATE, prot=mmap.PROT_READ | mmap.PROT_WRITE, ) if not single: self._map_segments() if barrier is not None: barriers_remaining -= 1 barrier() if self._creator: self._unlink() self._creator = False logger.info("Unlinked mmap file %s", self.mmap_path) if populate_shard is not None: populate_write_fn = _get_populate_write_fn(self.mmap_obj) index, count = populate_shard share = round_up(cdiv(self.total_size_bytes, count), self.page_size) start = min(index * share, self.total_size_bytes) if start < self.total_size_bytes: length = min(share, self.total_size_bytes - start) populate_write_fn(self.mmap_obj, start, length) if barrier is not None: barriers_remaining -= 1 barrier() except Exception: if self._creator: self._unlink() self._creator = False if hasattr(self, "mmap_obj") and self.mmap_obj is not None: # The prefault traceback may retain an exported buffer. with contextlib.suppress(BufferError): self.mmap_obj.close() self.mmap_obj = None if hasattr(self, "fd") and self.fd is not None: os.close(self.fd) self.fd = None # Join remaining barriers so peers do not wait until the collective # times out. A barrier failure must not replace the setup error. if barrier is not None: for _ in range(barriers_remaining): try: barrier() except Exception: logger.warning( "Failed to release peers waiting at the mmap barrier", exc_info=True, ) raise self._base = torch.frombuffer(memoryview(self.mmap_obj), dtype=torch.int8) self._views: list[torch.Tensor] = [] self._canonical_offset = 0 self.is_pinned: bool = False self.pinned_addresses: list[int] = [] if populate_shard is not None: return populate_write_fn = _get_populate_write_fn(self.mmap_obj) if rank is not None: # Populate only this worker's pages (one slot per chunk row). _t0 = time.perf_counter() for offset, length in self.iter_worker_slot_ranges(): populate_write_fn(self.mmap_obj, offset, length) logger.debug( "MADV_POPULATE_WRITE loop: %d chunks in %.3f s", num_chunks, time.perf_counter() - _t0, ) else: # No rank — populate the entire shared region in one call. _t0 = time.perf_counter() populate_write_fn(self.mmap_obj, 0, self.total_size_bytes) logger.debug( "MADV_POPULATE_WRITE entire region: %.3f s", time.perf_counter() - _t0 ) def _map_segments(self) -> None: """Replace the anonymous mapping with the segment files, back to back.""" assert self.mmap_obj is not None and self.fd is not None base = ctypes.addressof(ctypes.c_char.from_buffer(self.mmap_obj)) for path, offset, length in self._segments: fd = self.fd if offset == 0 else os.open(path, os.O_RDWR) address = _libc_mmap()( base + offset, length, mmap.PROT_READ | mmap.PROT_WRITE, mmap.MAP_SHARED | _MAP_FIXED, fd, 0, ) if fd != self.fd: os.close(fd) if address != base + offset: raise OSError(ctypes.get_errno(), f"Failed to map {path}") def _unlink(self) -> None: for path, _, _ in self._segments: with contextlib.suppress(FileNotFoundError): os.unlink(path) def iter_worker_slot_ranges(self) -> Iterator[tuple[int, int]]: """Yield this worker's slots as page-aligned (offset, length) ranges.""" assert self.rank is not None worker_offset = self.rank * self.cpu_page_size first = 0 if len(self._segments) > 1: # Start each worker on a different file. first = ( self.rank * self.num_chunks // (self._row_stride // self.cpu_page_size) ) for i in range(self.num_chunks): row = (first + i) % self.num_chunks start = row * self._row_stride + worker_offset aligned = start - start % self.page_size end = round_up(start + self.cpu_page_size, self.page_size) yield aligned, end - aligned @property def base_tensor(self) -> torch.Tensor: if self._base is None: raise RuntimeError("Shared offload region has been released.") return self._base def create_next_worker_view(self, tensor_page_size: int) -> torch.Tensor: """Allocate a strided int8 view for this worker, one canonical tensor. Must be called once per canonical tensor. The full mmap layout is: worker0_chunk0 | worker1_chunk0 | ... | worker{M-1}_chunk0 worker0_chunk1 | worker1_chunk1 | ... | worker{M-1}_chunk1 ... Each worker_chunk cell is cpu_page_size bytes and holds all canonical tensors for that worker and chunk concatenated: [ tensor0_data | tensor1_data | ... | tensor{L-1}_data ] Consecutive rows are separated by row_stride = cpu_page_size * M. Returns an int8 tensor of shape (num_chunks, tensor_page_size) with stride (row_stride, 1). Using int8 keeps stride == bytes, so swap_blocks address arithmetic works without any dtype conversion. Args: tensor_page_size: Bytes per chunk for this tensor. """ assert self.rank is not None new_offset = self._worker_offset + tensor_page_size assert new_offset <= self._worker_area_end, ( f"Worker offset {new_offset} exceeds worker area end " f"{self._worker_area_end} (overflowed by " f"{new_offset - self._worker_area_end} bytes)" ) worker_layer_view = torch.as_strided( self._base, size=(self.num_chunks, tensor_page_size), stride=(self._row_stride, 1), storage_offset=self._worker_offset, ) self._worker_offset = new_offset self._views.append(worker_layer_view) return worker_layer_view def create_next_canonical_view(self, tensor_page_size: int) -> torch.Tensor: """Allocate a strided int8 view shared by all workers for one canonical tensor (canonical layout). Must be called once per canonical tensor, instead of create_next_worker_view. The full mmap layout is: |<-------- canonical area ------->|<-------- unused ------->| | all workers share this area | | | | | | [ canonical_t0 | canonical_t1 ] | | | [ canonical_t0 | canonical_t1 ] | | | [ canonical_t0 | canonical_t1 ] | | ^ ^ _canonical_offset=0, then advances by each tensor's size Each canonical_t{i} cell is that tensor's canonical page for the chunk. Canonical areas are carved consecutively from the start of each chunk row; consecutive rows are separated by row_stride. Every worker gets the identical byte ranges and writes only its disjoint bytes within them, as described by its canonical mappings — unlike create_next_worker_view, which gives each worker a private cpu_page_size slot per row. The trailing unused bytes exist only when the canonical pages sum to less than row_stride: page-alignment padding of the row, or deduplication of KV replicated across workers (e.g. the MLA latent), where one canonical copy replaces world_size worker copies. Args: tensor_page_size: Canonical bytes per chunk for this tensor. """ new_offset = self._canonical_offset + tensor_page_size assert new_offset <= self._row_stride view = torch.as_strided( self._base, size=(self.num_chunks, tensor_page_size), stride=(self._row_stride, 1), storage_offset=self._canonical_offset, ) self._canonical_offset = new_offset self._views.append(view) return view def create_kv_memoryview(self) -> memoryview: """Return a zero-copy memoryview over the entire KV buffer. Shape: (num_chunks, row_stride_bytes). Secondary tiers address chunk *b* as ``view[b]``. """ kv_tensor = self._base.view(self.num_chunks, self._row_stride) np_arr = kv_tensor.numpy() assert np_arr.ctypes.data == self._base.data_ptr(), ( "view()/numpy() created a copy instead of sharing the mmap buffer; " "secondary tiers require zero-copy access to primary KV data" ) return memoryview(np_arr) def cleanup(self) -> None: if self.is_pinned and self._base is not None: base_ptr = self._base.data_ptr() addresses = self.pinned_addresses or [base_ptr] for address in reversed(addresses): host_unregister(address) self.pinned_addresses.clear() self.is_pinned = False # Release views before _base: each view holds a _base reference and a # direct StorageImpl reference. Freeing views first lets both refcounts # drop so the storage (which holds the mmap_obj buffer export) is freed # before mmap_obj.close() is called below. if self._views is not None: self._views.clear() self._base = None if self.mmap_obj: retained = False try: self.mmap_obj.close() except BufferError: # A native user still exports this mapping, so unmapping would # let the NIC write into reused address space. Keep it mapped; # dropping the fd and the pathname below is still safe. retained = True logger.warning( "Keeping %s mapped: buffers are still exported to native users", self.mmap_path, ) except Exception: logger.warning("Failed to close mmap_obj", exc_info=True) if not retained: self.mmap_obj = None if self.fd is not None: try: os.close(self.fd) except Exception: logger.warning("Failed to close fd %s", self.fd, exc_info=True) self.fd = None if self._creator and getattr(self, "mmap_path", None): try: self._unlink() logger.info("Removed mmap file %s", self.mmap_path) except Exception: logger.warning( "Failed to unlink path %s", self.mmap_path, exc_info=True ) self._creator = False