diff --git a/tests/v1/kv_offload/cpu/test_shared_offload_region.py b/tests/v1/kv_offload/cpu/test_shared_offload_region.py index dd6debfd43..7b5867ddb7 100644 --- a/tests/v1/kv_offload/cpu/test_shared_offload_region.py +++ b/tests/v1/kv_offload/cpu/test_shared_offload_region.py @@ -3,6 +3,8 @@ """Unit tests for SharedOffloadRegion.""" import contextlib +import ctypes +import errno import mmap import os import threading @@ -57,6 +59,29 @@ def _cleanup_file(path: str) -> None: os.unlink(path) +def _page_residency(mmap_obj: mmap.mmap, length: int) -> list[bool]: + """Return Linux page-residency bits for a writable mmap.""" + try: + libc = ctypes.CDLL(None, use_errno=True) + mincore = libc.mincore + except AttributeError: + pytest.skip("mincore is unavailable") + + page_count = (length + PAGE_SIZE - 1) // PAGE_SIZE + mincore.argtypes = [ + ctypes.c_void_p, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_ubyte), + ] + mincore.restype = ctypes.c_int + vector = (ctypes.c_ubyte * page_count)() + address = ctypes.addressof(ctypes.c_ubyte.from_buffer(mmap_obj)) + result = mincore(ctypes.c_void_p(address), length, vector) + if result != 0: + raise OSError(ctypes.get_errno()) + return [bool(value & 1) for value in vector] + + @contextlib.contextmanager def _region(engine_id: str, **kwargs): """Context manager: create one region, clean up on exit.""" @@ -405,6 +430,139 @@ def test_file_has_correct_size(iid): assert os.path.getsize(r.mmap_path) == 4 * PAGE_SIZE +def test_madvise_success_selects_madvise_population(iid, monkeypatch): + """A successful probe must keep using MADV_POPULATE_WRITE — the fallback + helper must never be invoked on a kernel that accepts the advice. + + Spies on both helpers so we can verify call counts: 1 probe call + + N populate calls, fallback 0 times. + """ + from vllm.v1.kv_offload.cpu import shared_offload_region as sor + + madvise_calls: list[tuple[int, int, int]] = [] + fallback_calls: list[tuple[int, int]] = [] + + def _spy_madvise(mm, off, ln): + madvise_calls.append((off, ln, id(mm))) + + def _spy_fallback(mm, off, ln): + fallback_calls.append((off, ln)) + + monkeypatch.setattr(sor, "_madvise_populate_write", _spy_madvise) + monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback) + + num_blocks = 3 + num_workers = 2 + with _region(iid, num_blocks=num_blocks, num_workers=num_workers, rank=1): + # 1 probe (PAGESIZE) + N populate calls (one per block per worker column). + expected_populate = num_blocks # ranked path: one call per block + mmap_id = madvise_calls[0][2] + assert madvise_calls == [ + (0, mmap.PAGESIZE, mmap_id), + (mmap.PAGESIZE, mmap.PAGESIZE, mmap_id), + (3 * mmap.PAGESIZE, mmap.PAGESIZE, mmap_id), + (5 * mmap.PAGESIZE, mmap.PAGESIZE, mmap_id), + ] + assert len(madvise_calls) == 1 + expected_populate + assert fallback_calls == [], ( + "native-path constructor must not invoke the fallback helper" + ) + + +def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch): + """An EINVAL probe must select fallback for every ranked block.""" + from vllm.v1.kv_offload.cpu import shared_offload_region as sor + + fallback_calls: list[tuple[int, int]] = [] + real_fallback = sor._fallback_populate_write + + def _raise_einval(mm, off, ln): + raise OSError(errno.EINVAL, "simulated unsupported kernel") + + def _spy_fallback(mm, off, ln): + fallback_calls.append((off, ln)) + return real_fallback(mm, off, ln) + + monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval) + monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback) + + with _region(iid, num_blocks=3, num_workers=2, rank=1): + assert fallback_calls == [ + (mmap.PAGESIZE, mmap.PAGESIZE), + (3 * mmap.PAGESIZE, mmap.PAGESIZE), + (5 * mmap.PAGESIZE, mmap.PAGESIZE), + ] + + +def test_madvise_einval_selects_fallback_for_unranked_region(iid, monkeypatch): + """An EINVAL probe must select fallback for the whole unranked region.""" + from vllm.v1.kv_offload.cpu import shared_offload_region as sor + + fallback_calls: list[tuple[int, int]] = [] + real_fallback = sor._fallback_populate_write + + def _raise_einval(mm, off, ln): + raise OSError(errno.EINVAL, "simulated unsupported kernel") + + def _spy_fallback(mm, off, ln): + fallback_calls.append((off, ln)) + return real_fallback(mm, off, ln) + + monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval) + monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback) + + with _region(iid, num_blocks=3, num_workers=2, rank=None): + assert fallback_calls == [(0, 6 * mmap.PAGESIZE)] + + +def test_fallback_populate_write_preserves_bytes_and_faults_pages(): + """Fallback preserves existing bytes and touches every target page.""" + from vllm.v1.kv_offload.cpu import shared_offload_region as sor + + size = 3 * mmap.PAGESIZE + if not hasattr(mmap, "MADV_DONTNEED"): + pytest.skip("MADV_DONTNEED is unavailable") + + mmap_obj = mmap.mmap( + -1, + size, + flags=mmap.MAP_SHARED, + prot=mmap.PROT_READ | mmap.PROT_WRITE, + ) + try: + mmap_obj.madvise(mmap.MADV_DONTNEED, 0, size) + if any(_page_residency(mmap_obj, size)): + pytest.skip("kernel did not discard anonymous mmap pages") + + mmap_obj[0] = 0xAB + assert _page_residency(mmap_obj, size) == [True, False, False] + + sor._fallback_populate_write(mmap_obj, 0, size) + + assert _page_residency(mmap_obj, size) == [True, True, True] + assert mmap_obj[0] == 0xAB + finally: + mmap_obj.close() + + +def test_madvise_unexpected_oserror_propagates(iid, monkeypatch): + """Only EINVAL triggers the fallback. Other OSErrors (e.g. EIO) must + propagate out of __init__, not be silently masked by the fallback branch. + """ + from vllm.v1.kv_offload.cpu import shared_offload_region as sor + + monkeypatch.setattr( + sor, + "_madvise_populate_write", + lambda mm, off, ln: (_ for _ in ()).throw( + OSError(errno.EIO, "simulated I/O failure") + ), + ) + with pytest.raises(OSError) as exc_info: + _make_region(iid) + assert exc_info.value.errno == errno.EIO + + # --------------------------------------------------------------------------- # Multi-worker race — concurrent construction # --------------------------------------------------------------------------- diff --git a/vllm/v1/kv_offload/cpu/shared_offload_region.py b/vllm/v1/kv_offload/cpu/shared_offload_region.py index 8456f37c71..69cec67bff 100644 --- a/vllm/v1/kv_offload/cpu/shared_offload_region.py +++ b/vllm/v1/kv_offload/cpu/shared_offload_region.py @@ -1,9 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import errno import mmap import os import time +from collections.abc import Callable +import numpy as np import torch from vllm.distributed.device_communicators.shm_broadcast import ( @@ -14,6 +17,9 @@ from vllm.platforms import current_platform logger = init_logger(__name__) +# MADV_POPULATE_WRITE was added in Linux 5.14 (value 23). +_MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23) + def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0) -> None: """Spin-wait until the file reaches expected_size (creator truncated it).""" @@ -28,6 +34,35 @@ def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0) -> N time.sleep(0.005) +def _madvise_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None: + mmap_obj.madvise(_MADV_POPULATE_WRITE, offset, length) + + +def _fallback_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None: + # Touch one byte per page via a read-modify-write so existing bytes are + # preserved — a peer worker may have already written KV data into this + # shared mmap by the time we run on a kernel without MADV_POPULATE_WRITE. + arr = np.frombuffer(mmap_obj, dtype=np.uint8) + arr[offset : offset + length : mmap.PAGESIZE] |= 0 + + +def _get_populate_write_fn( + mmap_obj: mmap.mmap, +) -> Callable[[mmap.mmap, int, int], None]: + """Select the pre-faulting method once for this mmap.""" + try: + _madvise_populate_write(mmap_obj, 0, mmap.PAGESIZE) + except OSError as e: + if e.errno != errno.EINVAL: + raise + logger.warning( + "MADV_POPULATE_WRITE is not supported; falling back to per-page " + "writes for mmap pre-population. Startup may be slower." + ) + return _fallback_populate_write + return _madvise_populate_write + + class SharedOffloadRegion: """ Single mmap-backed memory region shared across all workers for a @@ -104,8 +139,8 @@ class SharedOffloadRegion: prot=mmap.PROT_READ | mmap.PROT_WRITE, ) - # MADV_POPULATE_WRITE was added in Linux 5.14 (value 23). - _MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23) + populate_write_fn = _get_populate_write_fn(self.mmap_obj) + if rank is not None: # Populate only this worker's pages (one slot per block row). worker_offset = rank * cpu_page_size @@ -116,9 +151,7 @@ class SharedOffloadRegion: aligned_offset = (raw_offset // page_size) * page_size end = raw_offset + cpu_page_size aligned_length = end - aligned_offset - self.mmap_obj.madvise( - _MADV_POPULATE_WRITE, aligned_offset, aligned_length - ) + populate_write_fn(self.mmap_obj, aligned_offset, aligned_length) logger.debug( "MADV_POPULATE_WRITE loop: %d blocks in %.3f s", num_blocks, @@ -127,7 +160,7 @@ class SharedOffloadRegion: else: # No rank — populate the entire shared region in one call. _t0 = time.perf_counter() - self.mmap_obj.madvise(_MADV_POPULATE_WRITE, 0, self.total_size_bytes) + populate_write_fn(self.mmap_obj, 0, self.total_size_bytes) logger.debug( "MADV_POPULATE_WRITE entire region: %.3f s", time.perf_counter() - _t0 )