[Bugfix][KV Offload] Fall back when MADV_POPULATE_WRITE is unsupported (#51116)

Signed-off-by: Guy Girmonsky <guygir@gmail.com>
Signed-off-by: Alex <jihui.huang@daocloud.io>
Co-authored-by: Guy Girmonsky <guygir@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
AlexHuang
2026-08-06 13:48:01 +03:00
committed by GitHub
co-authored by Guy Girmonsky Claude Opus 5 OpenAI Codex
parent 22013f74ff
commit ef2615c2e0
2 changed files with 197 additions and 6 deletions
@@ -3,6 +3,8 @@
"""Unit tests for SharedOffloadRegion."""
import contextlib
import ctypes
import errno
import mmap
import os
import threading
@@ -57,6 +59,29 @@ def _cleanup_file(path: str) -> None:
os.unlink(path)
def _page_residency(mmap_obj: mmap.mmap, length: int) -> list[bool]:
"""Return Linux page-residency bits for a writable mmap."""
try:
libc = ctypes.CDLL(None, use_errno=True)
mincore = libc.mincore
except AttributeError:
pytest.skip("mincore is unavailable")
page_count = (length + PAGE_SIZE - 1) // PAGE_SIZE
mincore.argtypes = [
ctypes.c_void_p,
ctypes.c_size_t,
ctypes.POINTER(ctypes.c_ubyte),
]
mincore.restype = ctypes.c_int
vector = (ctypes.c_ubyte * page_count)()
address = ctypes.addressof(ctypes.c_ubyte.from_buffer(mmap_obj))
result = mincore(ctypes.c_void_p(address), length, vector)
if result != 0:
raise OSError(ctypes.get_errno())
return [bool(value & 1) for value in vector]
@contextlib.contextmanager
def _region(engine_id: str, **kwargs):
"""Context manager: create one region, clean up on exit."""
@@ -405,6 +430,139 @@ def test_file_has_correct_size(iid):
assert os.path.getsize(r.mmap_path) == 4 * PAGE_SIZE
def test_madvise_success_selects_madvise_population(iid, monkeypatch):
"""A successful probe must keep using MADV_POPULATE_WRITE — the fallback
helper must never be invoked on a kernel that accepts the advice.
Spies on both helpers so we can verify call counts: 1 probe call +
N populate calls, fallback 0 times.
"""
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
madvise_calls: list[tuple[int, int, int]] = []
fallback_calls: list[tuple[int, int]] = []
def _spy_madvise(mm, off, ln):
madvise_calls.append((off, ln, id(mm)))
def _spy_fallback(mm, off, ln):
fallback_calls.append((off, ln))
monkeypatch.setattr(sor, "_madvise_populate_write", _spy_madvise)
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
num_blocks = 3
num_workers = 2
with _region(iid, num_blocks=num_blocks, num_workers=num_workers, rank=1):
# 1 probe (PAGESIZE) + N populate calls (one per block per worker column).
expected_populate = num_blocks # ranked path: one call per block
mmap_id = madvise_calls[0][2]
assert madvise_calls == [
(0, mmap.PAGESIZE, mmap_id),
(mmap.PAGESIZE, mmap.PAGESIZE, mmap_id),
(3 * mmap.PAGESIZE, mmap.PAGESIZE, mmap_id),
(5 * mmap.PAGESIZE, mmap.PAGESIZE, mmap_id),
]
assert len(madvise_calls) == 1 + expected_populate
assert fallback_calls == [], (
"native-path constructor must not invoke the fallback helper"
)
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
"""An EINVAL probe must select fallback for every ranked block."""
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
fallback_calls: list[tuple[int, int]] = []
real_fallback = sor._fallback_populate_write
def _raise_einval(mm, off, ln):
raise OSError(errno.EINVAL, "simulated unsupported kernel")
def _spy_fallback(mm, off, ln):
fallback_calls.append((off, ln))
return real_fallback(mm, off, ln)
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
with _region(iid, num_blocks=3, num_workers=2, rank=1):
assert fallback_calls == [
(mmap.PAGESIZE, mmap.PAGESIZE),
(3 * mmap.PAGESIZE, mmap.PAGESIZE),
(5 * mmap.PAGESIZE, mmap.PAGESIZE),
]
def test_madvise_einval_selects_fallback_for_unranked_region(iid, monkeypatch):
"""An EINVAL probe must select fallback for the whole unranked region."""
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
fallback_calls: list[tuple[int, int]] = []
real_fallback = sor._fallback_populate_write
def _raise_einval(mm, off, ln):
raise OSError(errno.EINVAL, "simulated unsupported kernel")
def _spy_fallback(mm, off, ln):
fallback_calls.append((off, ln))
return real_fallback(mm, off, ln)
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
with _region(iid, num_blocks=3, num_workers=2, rank=None):
assert fallback_calls == [(0, 6 * mmap.PAGESIZE)]
def test_fallback_populate_write_preserves_bytes_and_faults_pages():
"""Fallback preserves existing bytes and touches every target page."""
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
size = 3 * mmap.PAGESIZE
if not hasattr(mmap, "MADV_DONTNEED"):
pytest.skip("MADV_DONTNEED is unavailable")
mmap_obj = mmap.mmap(
-1,
size,
flags=mmap.MAP_SHARED,
prot=mmap.PROT_READ | mmap.PROT_WRITE,
)
try:
mmap_obj.madvise(mmap.MADV_DONTNEED, 0, size)
if any(_page_residency(mmap_obj, size)):
pytest.skip("kernel did not discard anonymous mmap pages")
mmap_obj[0] = 0xAB
assert _page_residency(mmap_obj, size) == [True, False, False]
sor._fallback_populate_write(mmap_obj, 0, size)
assert _page_residency(mmap_obj, size) == [True, True, True]
assert mmap_obj[0] == 0xAB
finally:
mmap_obj.close()
def test_madvise_unexpected_oserror_propagates(iid, monkeypatch):
"""Only EINVAL triggers the fallback. Other OSErrors (e.g. EIO) must
propagate out of __init__, not be silently masked by the fallback branch.
"""
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
monkeypatch.setattr(
sor,
"_madvise_populate_write",
lambda mm, off, ln: (_ for _ in ()).throw(
OSError(errno.EIO, "simulated I/O failure")
),
)
with pytest.raises(OSError) as exc_info:
_make_region(iid)
assert exc_info.value.errno == errno.EIO
# ---------------------------------------------------------------------------
# Multi-worker race — concurrent construction
# ---------------------------------------------------------------------------
@@ -1,9 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import errno
import mmap
import os
import time
from collections.abc import Callable
import numpy as np
import torch
from vllm.distributed.device_communicators.shm_broadcast import (
@@ -14,6 +17,9 @@ from vllm.platforms import current_platform
logger = init_logger(__name__)
# MADV_POPULATE_WRITE was added in Linux 5.14 (value 23).
_MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23)
def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0) -> None:
"""Spin-wait until the file reaches expected_size (creator truncated it)."""
@@ -28,6 +34,35 @@ def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0) -> N
time.sleep(0.005)
def _madvise_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None:
mmap_obj.madvise(_MADV_POPULATE_WRITE, offset, length)
def _fallback_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None:
# Touch one byte per page via a read-modify-write so existing bytes are
# preserved — a peer worker may have already written KV data into this
# shared mmap by the time we run on a kernel without MADV_POPULATE_WRITE.
arr = np.frombuffer(mmap_obj, dtype=np.uint8)
arr[offset : offset + length : mmap.PAGESIZE] |= 0
def _get_populate_write_fn(
mmap_obj: mmap.mmap,
) -> Callable[[mmap.mmap, int, int], None]:
"""Select the pre-faulting method once for this mmap."""
try:
_madvise_populate_write(mmap_obj, 0, mmap.PAGESIZE)
except OSError as e:
if e.errno != errno.EINVAL:
raise
logger.warning(
"MADV_POPULATE_WRITE is not supported; falling back to per-page "
"writes for mmap pre-population. Startup may be slower."
)
return _fallback_populate_write
return _madvise_populate_write
class SharedOffloadRegion:
"""
Single mmap-backed memory region shared across all workers for a
@@ -104,8 +139,8 @@ class SharedOffloadRegion:
prot=mmap.PROT_READ | mmap.PROT_WRITE,
)
# MADV_POPULATE_WRITE was added in Linux 5.14 (value 23).
_MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23)
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
if rank is not None:
# Populate only this worker's pages (one slot per block row).
worker_offset = rank * cpu_page_size
@@ -116,9 +151,7 @@ class SharedOffloadRegion:
aligned_offset = (raw_offset // page_size) * page_size
end = raw_offset + cpu_page_size
aligned_length = end - aligned_offset
self.mmap_obj.madvise(
_MADV_POPULATE_WRITE, aligned_offset, aligned_length
)
populate_write_fn(self.mmap_obj, aligned_offset, aligned_length)
logger.debug(
"MADV_POPULATE_WRITE loop: %d blocks in %.3f s",
num_blocks,
@@ -127,7 +160,7 @@ class SharedOffloadRegion:
else:
# No rank — populate the entire shared region in one call.
_t0 = time.perf_counter()
self.mmap_obj.madvise(_MADV_POPULATE_WRITE, 0, self.total_size_bytes)
populate_write_fn(self.mmap_obj, 0, self.total_size_bytes)
logger.debug(
"MADV_POPULATE_WRITE entire region: %.3f s", time.perf_counter() - _t0
)