[Bugfix][KV Offload] Fall back when MADV_POPULATE_WRITE is unsupported (#51116)
Signed-off-by: Guy Girmonsky <guygir@gmail.com> Signed-off-by: Alex <jihui.huang@daocloud.io> Co-authored-by: Guy Girmonsky <guygir@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Guy Girmonsky
Claude Opus 5
OpenAI Codex
parent
22013f74ff
commit
ef2615c2e0
@@ -3,6 +3,8 @@
|
||||
"""Unit tests for SharedOffloadRegion."""
|
||||
|
||||
import contextlib
|
||||
import ctypes
|
||||
import errno
|
||||
import mmap
|
||||
import os
|
||||
import threading
|
||||
@@ -57,6 +59,29 @@ def _cleanup_file(path: str) -> None:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
def _page_residency(mmap_obj: mmap.mmap, length: int) -> list[bool]:
|
||||
"""Return Linux page-residency bits for a writable mmap."""
|
||||
try:
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
mincore = libc.mincore
|
||||
except AttributeError:
|
||||
pytest.skip("mincore is unavailable")
|
||||
|
||||
page_count = (length + PAGE_SIZE - 1) // PAGE_SIZE
|
||||
mincore.argtypes = [
|
||||
ctypes.c_void_p,
|
||||
ctypes.c_size_t,
|
||||
ctypes.POINTER(ctypes.c_ubyte),
|
||||
]
|
||||
mincore.restype = ctypes.c_int
|
||||
vector = (ctypes.c_ubyte * page_count)()
|
||||
address = ctypes.addressof(ctypes.c_ubyte.from_buffer(mmap_obj))
|
||||
result = mincore(ctypes.c_void_p(address), length, vector)
|
||||
if result != 0:
|
||||
raise OSError(ctypes.get_errno())
|
||||
return [bool(value & 1) for value in vector]
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _region(engine_id: str, **kwargs):
|
||||
"""Context manager: create one region, clean up on exit."""
|
||||
@@ -405,6 +430,139 @@ def test_file_has_correct_size(iid):
|
||||
assert os.path.getsize(r.mmap_path) == 4 * PAGE_SIZE
|
||||
|
||||
|
||||
def test_madvise_success_selects_madvise_population(iid, monkeypatch):
|
||||
"""A successful probe must keep using MADV_POPULATE_WRITE — the fallback
|
||||
helper must never be invoked on a kernel that accepts the advice.
|
||||
|
||||
Spies on both helpers so we can verify call counts: 1 probe call +
|
||||
N populate calls, fallback 0 times.
|
||||
"""
|
||||
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
||||
|
||||
madvise_calls: list[tuple[int, int, int]] = []
|
||||
fallback_calls: list[tuple[int, int]] = []
|
||||
|
||||
def _spy_madvise(mm, off, ln):
|
||||
madvise_calls.append((off, ln, id(mm)))
|
||||
|
||||
def _spy_fallback(mm, off, ln):
|
||||
fallback_calls.append((off, ln))
|
||||
|
||||
monkeypatch.setattr(sor, "_madvise_populate_write", _spy_madvise)
|
||||
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
||||
|
||||
num_blocks = 3
|
||||
num_workers = 2
|
||||
with _region(iid, num_blocks=num_blocks, num_workers=num_workers, rank=1):
|
||||
# 1 probe (PAGESIZE) + N populate calls (one per block per worker column).
|
||||
expected_populate = num_blocks # ranked path: one call per block
|
||||
mmap_id = madvise_calls[0][2]
|
||||
assert madvise_calls == [
|
||||
(0, mmap.PAGESIZE, mmap_id),
|
||||
(mmap.PAGESIZE, mmap.PAGESIZE, mmap_id),
|
||||
(3 * mmap.PAGESIZE, mmap.PAGESIZE, mmap_id),
|
||||
(5 * mmap.PAGESIZE, mmap.PAGESIZE, mmap_id),
|
||||
]
|
||||
assert len(madvise_calls) == 1 + expected_populate
|
||||
assert fallback_calls == [], (
|
||||
"native-path constructor must not invoke the fallback helper"
|
||||
)
|
||||
|
||||
|
||||
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
|
||||
"""An EINVAL probe must select fallback for every ranked block."""
|
||||
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
||||
|
||||
fallback_calls: list[tuple[int, int]] = []
|
||||
real_fallback = sor._fallback_populate_write
|
||||
|
||||
def _raise_einval(mm, off, ln):
|
||||
raise OSError(errno.EINVAL, "simulated unsupported kernel")
|
||||
|
||||
def _spy_fallback(mm, off, ln):
|
||||
fallback_calls.append((off, ln))
|
||||
return real_fallback(mm, off, ln)
|
||||
|
||||
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
|
||||
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
||||
|
||||
with _region(iid, num_blocks=3, num_workers=2, rank=1):
|
||||
assert fallback_calls == [
|
||||
(mmap.PAGESIZE, mmap.PAGESIZE),
|
||||
(3 * mmap.PAGESIZE, mmap.PAGESIZE),
|
||||
(5 * mmap.PAGESIZE, mmap.PAGESIZE),
|
||||
]
|
||||
|
||||
|
||||
def test_madvise_einval_selects_fallback_for_unranked_region(iid, monkeypatch):
|
||||
"""An EINVAL probe must select fallback for the whole unranked region."""
|
||||
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
||||
|
||||
fallback_calls: list[tuple[int, int]] = []
|
||||
real_fallback = sor._fallback_populate_write
|
||||
|
||||
def _raise_einval(mm, off, ln):
|
||||
raise OSError(errno.EINVAL, "simulated unsupported kernel")
|
||||
|
||||
def _spy_fallback(mm, off, ln):
|
||||
fallback_calls.append((off, ln))
|
||||
return real_fallback(mm, off, ln)
|
||||
|
||||
monkeypatch.setattr(sor, "_madvise_populate_write", _raise_einval)
|
||||
monkeypatch.setattr(sor, "_fallback_populate_write", _spy_fallback)
|
||||
|
||||
with _region(iid, num_blocks=3, num_workers=2, rank=None):
|
||||
assert fallback_calls == [(0, 6 * mmap.PAGESIZE)]
|
||||
|
||||
|
||||
def test_fallback_populate_write_preserves_bytes_and_faults_pages():
|
||||
"""Fallback preserves existing bytes and touches every target page."""
|
||||
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
||||
|
||||
size = 3 * mmap.PAGESIZE
|
||||
if not hasattr(mmap, "MADV_DONTNEED"):
|
||||
pytest.skip("MADV_DONTNEED is unavailable")
|
||||
|
||||
mmap_obj = mmap.mmap(
|
||||
-1,
|
||||
size,
|
||||
flags=mmap.MAP_SHARED,
|
||||
prot=mmap.PROT_READ | mmap.PROT_WRITE,
|
||||
)
|
||||
try:
|
||||
mmap_obj.madvise(mmap.MADV_DONTNEED, 0, size)
|
||||
if any(_page_residency(mmap_obj, size)):
|
||||
pytest.skip("kernel did not discard anonymous mmap pages")
|
||||
|
||||
mmap_obj[0] = 0xAB
|
||||
assert _page_residency(mmap_obj, size) == [True, False, False]
|
||||
|
||||
sor._fallback_populate_write(mmap_obj, 0, size)
|
||||
|
||||
assert _page_residency(mmap_obj, size) == [True, True, True]
|
||||
assert mmap_obj[0] == 0xAB
|
||||
finally:
|
||||
mmap_obj.close()
|
||||
|
||||
|
||||
def test_madvise_unexpected_oserror_propagates(iid, monkeypatch):
|
||||
"""Only EINVAL triggers the fallback. Other OSErrors (e.g. EIO) must
|
||||
propagate out of __init__, not be silently masked by the fallback branch.
|
||||
"""
|
||||
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
||||
|
||||
monkeypatch.setattr(
|
||||
sor,
|
||||
"_madvise_populate_write",
|
||||
lambda mm, off, ln: (_ for _ in ()).throw(
|
||||
OSError(errno.EIO, "simulated I/O failure")
|
||||
),
|
||||
)
|
||||
with pytest.raises(OSError) as exc_info:
|
||||
_make_region(iid)
|
||||
assert exc_info.value.errno == errno.EIO
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Multi-worker race — concurrent construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import errno
|
||||
import mmap
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
@@ -14,6 +17,9 @@ from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MADV_POPULATE_WRITE was added in Linux 5.14 (value 23).
|
||||
_MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23)
|
||||
|
||||
|
||||
def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0) -> None:
|
||||
"""Spin-wait until the file reaches expected_size (creator truncated it)."""
|
||||
@@ -28,6 +34,35 @@ def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0) -> N
|
||||
time.sleep(0.005)
|
||||
|
||||
|
||||
def _madvise_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None:
|
||||
mmap_obj.madvise(_MADV_POPULATE_WRITE, offset, length)
|
||||
|
||||
|
||||
def _fallback_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None:
|
||||
# Touch one byte per page via a read-modify-write so existing bytes are
|
||||
# preserved — a peer worker may have already written KV data into this
|
||||
# shared mmap by the time we run on a kernel without MADV_POPULATE_WRITE.
|
||||
arr = np.frombuffer(mmap_obj, dtype=np.uint8)
|
||||
arr[offset : offset + length : mmap.PAGESIZE] |= 0
|
||||
|
||||
|
||||
def _get_populate_write_fn(
|
||||
mmap_obj: mmap.mmap,
|
||||
) -> Callable[[mmap.mmap, int, int], None]:
|
||||
"""Select the pre-faulting method once for this mmap."""
|
||||
try:
|
||||
_madvise_populate_write(mmap_obj, 0, mmap.PAGESIZE)
|
||||
except OSError as e:
|
||||
if e.errno != errno.EINVAL:
|
||||
raise
|
||||
logger.warning(
|
||||
"MADV_POPULATE_WRITE is not supported; falling back to per-page "
|
||||
"writes for mmap pre-population. Startup may be slower."
|
||||
)
|
||||
return _fallback_populate_write
|
||||
return _madvise_populate_write
|
||||
|
||||
|
||||
class SharedOffloadRegion:
|
||||
"""
|
||||
Single mmap-backed memory region shared across all workers for a
|
||||
@@ -104,8 +139,8 @@ class SharedOffloadRegion:
|
||||
prot=mmap.PROT_READ | mmap.PROT_WRITE,
|
||||
)
|
||||
|
||||
# MADV_POPULATE_WRITE was added in Linux 5.14 (value 23).
|
||||
_MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23)
|
||||
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
|
||||
|
||||
if rank is not None:
|
||||
# Populate only this worker's pages (one slot per block row).
|
||||
worker_offset = rank * cpu_page_size
|
||||
@@ -116,9 +151,7 @@ class SharedOffloadRegion:
|
||||
aligned_offset = (raw_offset // page_size) * page_size
|
||||
end = raw_offset + cpu_page_size
|
||||
aligned_length = end - aligned_offset
|
||||
self.mmap_obj.madvise(
|
||||
_MADV_POPULATE_WRITE, aligned_offset, aligned_length
|
||||
)
|
||||
populate_write_fn(self.mmap_obj, aligned_offset, aligned_length)
|
||||
logger.debug(
|
||||
"MADV_POPULATE_WRITE loop: %d blocks in %.3f s",
|
||||
num_blocks,
|
||||
@@ -127,7 +160,7 @@ class SharedOffloadRegion:
|
||||
else:
|
||||
# No rank — populate the entire shared region in one call.
|
||||
_t0 = time.perf_counter()
|
||||
self.mmap_obj.madvise(_MADV_POPULATE_WRITE, 0, self.total_size_bytes)
|
||||
populate_write_fn(self.mmap_obj, 0, self.total_size_bytes)
|
||||
logger.debug(
|
||||
"MADV_POPULATE_WRITE entire region: %.3f s", time.perf_counter() - _t0
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user