[KV Offload][CI] Fall back to buffered I/O without O_DIRECT; fix flaky api-server test (#49734)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-07-24 13:56:57 -07:00
committed by GitHub
parent 84d26b9ee3
commit 89f6aa3a9e
5 changed files with 109 additions and 5 deletions
@@ -0,0 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Spawn worker targets kept free of heavy imports.
``multiprocessing`` with the ``spawn`` start method re-imports the module that
defines a process target in the child. Housing these stubs in a stdlib-only
module keeps child startup fast and deterministic, instead of paying a multi-
second ``import vllm`` before the child can run.
"""
def exit_before_report_worker(listen_address, sock, args, client_config=None):
"""Exit immediately without touching ``actual_address_pipe``."""
return
@@ -10,6 +10,9 @@ from unittest.mock import patch
import pytest
import zmq
from tests.entrypoints.unit_tests._api_server_spawn_workers import (
exit_before_report_worker,
)
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
from vllm.v1.utils import (
APIServerProcessManager,
@@ -228,6 +231,7 @@ def test_normal_completion(api_server_args):
def test_external_process_monitoring(api_server_args):
"""Test that wait_for_completion_or_failure handles additional processes."""
global WORKER_RUNTIME_SECONDS
prev_worker_runtime = WORKER_RUNTIME_SECONDS
WORKER_RUNTIME_SECONDS = 100
# Create and start the external process
@@ -307,6 +311,7 @@ def test_external_process_monitoring(api_server_args):
manager.shutdown()
mock_coordinator.shutdown()
time.sleep(0.2)
WORKER_RUNTIME_SECONDS = prev_worker_runtime
@pytest.mark.timeout(60)
@@ -383,10 +388,10 @@ def test_gather_actual_addresses_child_crash_before_report():
num_servers=num_servers,
input_addresses=placeholder_inputs,
output_addresses=placeholder_outputs,
# mock_run_api_server_worker exits without touching
# exit_before_report_worker exits without touching
# ``actual_address_pipe`` — simulates a child that dies before
# reporting its bound addresses.
target_server_fn=mock_run_api_server_worker,
target_server_fn=exit_before_report_worker,
)
try:
# Sentinel-first vs pipe-EOF-first both produce "reporting".
@@ -379,6 +379,43 @@ def test_store_load_data_integrity(fs_tier):
)
def test_store_load_roundtrip_without_o_direct(tmp_path, monkeypatch):
"""Buffered fallback must round-trip data when O_DIRECT is unsupported.
Simulates filesystems (e.g. overlayfs, some NFS) that reject O_DIRECT by
forcing the capability probe to report it unavailable.
"""
monkeypatch.setattr(
"vllm.v1.kv_offload.tiering.fs.manager.probe_o_direct",
lambda _dir: False,
)
tensor = _page_aligned_rand_tensor(4, _BLOCK_ELEMENTS)
tier = FileSystemTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=memoryview(tensor.numpy()),
tier_type="fs",
root_dir=str(tmp_path),
n_read_threads=4,
n_write_threads=4,
)
try:
assert tier._use_o_direct is False
keys = [key(0), key(1)]
expected = tensor[:2].clone()
tier.submit_store(make_job(1, keys, [0, 1]))
assert all(r.success for r in drain(tier))
tensor[:2] = 0.0
tier.submit_load(make_job(2, keys, [2, 3], is_promotion=True))
assert all(r.success for r in drain(tier))
for i, bid in enumerate([2, 3]):
assert torch.allclose(tensor[bid], expected[i])
finally:
tier.shutdown()
def test_wait_idle_blocks_until_tasks_complete():
"""wait_idle must not return while a task is still in flight."""
pool = DualQueueThreadPool(n_read_threads=1, n_write_threads=1)
+36 -2
View File
@@ -1,7 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import logging
import mmap
import os
import random
import threading
@@ -24,6 +26,34 @@ def _get_tmp_suffix() -> str:
return _thread_local.tmp_suffix
def probe_o_direct(directory: str) -> bool:
"""Return whether ``O_DIRECT`` I/O works in *directory*.
``O_DIRECT`` is unsupported on some filesystems (e.g. the overlayfs backing
a container ``/tmp``, older tmpfs, or some NFS mounts), where opening or
writing a file with it fails with ``EINVAL``. Probe once with an aligned
single-page write so callers can fall back to buffered I/O instead of
failing on every block.
"""
if not O_DIRECT:
return False
path = os.path.join(directory, f".o_direct_probe{_get_tmp_suffix()}")
page = mmap.mmap(-1, mmap.PAGESIZE)
try:
fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC | O_DIRECT, 0o644)
try:
os.write(fd, page)
finally:
os.close(fd)
return True
except OSError:
return False
finally:
page.close()
with contextlib.suppress(OSError):
os.remove(path)
def _ensure_dirs(path: str) -> None:
"""Create parent directories of *path* if they don't exist."""
os.makedirs(os.path.dirname(path), exist_ok=True)
@@ -34,6 +64,7 @@ def store_block(
buffer: memoryview,
offset: int,
block_size: int,
use_o_direct: bool = True,
) -> None:
"""
Store callback: Writes to a temp file then atomically replaces the destination.
@@ -49,10 +80,11 @@ def store_block(
# Write block atomically. Cast to a flat byte view so the slice uses byte
# indices; the raw memoryview may be multi-dimensional with itemsize > 1.
view_slice = buffer.cast("B")[offset : offset + block_size]
o_direct = O_DIRECT if use_o_direct else 0
try:
fd = os.open(
tmp_path,
os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC | O_DIRECT,
os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC | o_direct,
0o644,
)
try:
@@ -77,14 +109,16 @@ def load_block(
view: memoryview,
offset: int,
block_size: int,
use_o_direct: bool = True,
) -> None:
"""
Load callback: read one KV block from disk. Remove the file on failure.
"""
fd: int | None = None
view_slice = view.cast("B")[offset : offset + block_size]
o_direct = O_DIRECT if use_o_direct else 0
try:
fd = os.open(source_path, os.O_RDONLY | O_DIRECT)
fd = os.open(source_path, os.O_RDONLY | o_direct)
bytes_read = os.readv(fd, [view_slice])
if bytes_read < block_size:
raise OSError(f"Short read: expected {block_size} bytes, read {bytes_read}")
+15 -1
View File
@@ -49,7 +49,7 @@ from vllm.v1.kv_offload.tiering.base import (
ScheduleEndContext,
SecondaryTierManager,
)
from vllm.v1.kv_offload.tiering.fs.io import load_block, store_block
from vllm.v1.kv_offload.tiering.fs.io import load_block, probe_o_direct, store_block
from vllm.v1.kv_offload.tiering.fs.thread_pool import DualQueueThreadPool
if TYPE_CHECKING:
@@ -168,6 +168,18 @@ class FileSystemTierManager(SecondaryTierManager):
self.file_mapper.get_run_config(), f, indent=2, sort_keys=True
)
# Prefer O_DIRECT to bypass the page cache, but fall back to buffered
# I/O on filesystems that reject it (e.g. overlayfs, some NFS mounts)
# rather than failing every block.
self._use_o_direct = probe_o_direct(os.path.dirname(config_path))
if not self._use_o_direct:
logger.warning(
"O_DIRECT is not supported at '%s'; falling back to buffered "
"I/O for the '%s' KV offload tier.",
root_dir,
tier_type,
)
self._pool = DualQueueThreadPool(
n_read_threads,
n_write_threads,
@@ -198,6 +210,7 @@ class FileSystemTierManager(SecondaryTierManager):
self._primary_kv_view,
int(bid) * self._block_size,
self._block_size,
self._use_o_direct,
)
for key, bid in zip(job_metadata.keys, job_metadata.block_ids)
)
@@ -212,6 +225,7 @@ class FileSystemTierManager(SecondaryTierManager):
self._primary_kv_view,
int(bid) * self._block_size,
self._block_size,
self._use_o_direct,
)
for key, bid in zip(job_metadata.keys, job_metadata.block_ids)
)