From 89f6aa3a9e56a9228f55e8d722f23594223187cf Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:56:57 +0100 Subject: [PATCH] [KV Offload][CI] Fall back to buffered I/O without O_DIRECT; fix flaky api-server test (#49734) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- .../unit_tests/_api_server_spawn_workers.py | 14 +++++++ .../test_api_server_process_manager.py | 9 ++++- tests/v1/kv_offload/tiering/test_fs_tier.py | 37 ++++++++++++++++++ vllm/v1/kv_offload/tiering/fs/io.py | 38 ++++++++++++++++++- vllm/v1/kv_offload/tiering/fs/manager.py | 16 +++++++- 5 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 tests/entrypoints/unit_tests/_api_server_spawn_workers.py diff --git a/tests/entrypoints/unit_tests/_api_server_spawn_workers.py b/tests/entrypoints/unit_tests/_api_server_spawn_workers.py new file mode 100644 index 0000000000..c8ff969a8a --- /dev/null +++ b/tests/entrypoints/unit_tests/_api_server_spawn_workers.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Spawn worker targets kept free of heavy imports. + +``multiprocessing`` with the ``spawn`` start method re-imports the module that +defines a process target in the child. Housing these stubs in a stdlib-only +module keeps child startup fast and deterministic, instead of paying a multi- +second ``import vllm`` before the child can run. +""" + + +def exit_before_report_worker(listen_address, sock, args, client_config=None): + """Exit immediately without touching ``actual_address_pipe``.""" + return diff --git a/tests/entrypoints/unit_tests/test_api_server_process_manager.py b/tests/entrypoints/unit_tests/test_api_server_process_manager.py index ada7a8797f..902d0ce2d7 100644 --- a/tests/entrypoints/unit_tests/test_api_server_process_manager.py +++ b/tests/entrypoints/unit_tests/test_api_server_process_manager.py @@ -10,6 +10,9 @@ from unittest.mock import patch import pytest import zmq +from tests.entrypoints.unit_tests._api_server_spawn_workers import ( + exit_before_report_worker, +) from vllm.utils.network_utils import make_zmq_socket, split_zmq_path from vllm.v1.utils import ( APIServerProcessManager, @@ -228,6 +231,7 @@ def test_normal_completion(api_server_args): def test_external_process_monitoring(api_server_args): """Test that wait_for_completion_or_failure handles additional processes.""" global WORKER_RUNTIME_SECONDS + prev_worker_runtime = WORKER_RUNTIME_SECONDS WORKER_RUNTIME_SECONDS = 100 # Create and start the external process @@ -307,6 +311,7 @@ def test_external_process_monitoring(api_server_args): manager.shutdown() mock_coordinator.shutdown() time.sleep(0.2) + WORKER_RUNTIME_SECONDS = prev_worker_runtime @pytest.mark.timeout(60) @@ -383,10 +388,10 @@ def test_gather_actual_addresses_child_crash_before_report(): num_servers=num_servers, input_addresses=placeholder_inputs, output_addresses=placeholder_outputs, - # mock_run_api_server_worker exits without touching + # exit_before_report_worker exits without touching # ``actual_address_pipe`` — simulates a child that dies before # reporting its bound addresses. - target_server_fn=mock_run_api_server_worker, + target_server_fn=exit_before_report_worker, ) try: # Sentinel-first vs pipe-EOF-first both produce "reporting". diff --git a/tests/v1/kv_offload/tiering/test_fs_tier.py b/tests/v1/kv_offload/tiering/test_fs_tier.py index dcc92a4fa8..589d7dda77 100644 --- a/tests/v1/kv_offload/tiering/test_fs_tier.py +++ b/tests/v1/kv_offload/tiering/test_fs_tier.py @@ -379,6 +379,43 @@ def test_store_load_data_integrity(fs_tier): ) +def test_store_load_roundtrip_without_o_direct(tmp_path, monkeypatch): + """Buffered fallback must round-trip data when O_DIRECT is unsupported. + + Simulates filesystems (e.g. overlayfs, some NFS) that reject O_DIRECT by + forcing the capability probe to report it unavailable. + """ + monkeypatch.setattr( + "vllm.v1.kv_offload.tiering.fs.manager.probe_o_direct", + lambda _dir: False, + ) + tensor = _page_aligned_rand_tensor(4, _BLOCK_ELEMENTS) + tier = FileSystemTierManager( + offloading_spec=_MOCK_OFFLOADING_SPEC, + primary_kv_view=memoryview(tensor.numpy()), + tier_type="fs", + root_dir=str(tmp_path), + n_read_threads=4, + n_write_threads=4, + ) + try: + assert tier._use_o_direct is False + + keys = [key(0), key(1)] + expected = tensor[:2].clone() + tier.submit_store(make_job(1, keys, [0, 1])) + assert all(r.success for r in drain(tier)) + + tensor[:2] = 0.0 + tier.submit_load(make_job(2, keys, [2, 3], is_promotion=True)) + assert all(r.success for r in drain(tier)) + + for i, bid in enumerate([2, 3]): + assert torch.allclose(tensor[bid], expected[i]) + finally: + tier.shutdown() + + def test_wait_idle_blocks_until_tasks_complete(): """wait_idle must not return while a task is still in flight.""" pool = DualQueueThreadPool(n_read_threads=1, n_write_threads=1) diff --git a/vllm/v1/kv_offload/tiering/fs/io.py b/vllm/v1/kv_offload/tiering/fs/io.py index c5a82a73c6..7785591fd2 100644 --- a/vllm/v1/kv_offload/tiering/fs/io.py +++ b/vllm/v1/kv_offload/tiering/fs/io.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import contextlib import logging +import mmap import os import random import threading @@ -24,6 +26,34 @@ def _get_tmp_suffix() -> str: return _thread_local.tmp_suffix +def probe_o_direct(directory: str) -> bool: + """Return whether ``O_DIRECT`` I/O works in *directory*. + + ``O_DIRECT`` is unsupported on some filesystems (e.g. the overlayfs backing + a container ``/tmp``, older tmpfs, or some NFS mounts), where opening or + writing a file with it fails with ``EINVAL``. Probe once with an aligned + single-page write so callers can fall back to buffered I/O instead of + failing on every block. + """ + if not O_DIRECT: + return False + path = os.path.join(directory, f".o_direct_probe{_get_tmp_suffix()}") + page = mmap.mmap(-1, mmap.PAGESIZE) + try: + fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC | O_DIRECT, 0o644) + try: + os.write(fd, page) + finally: + os.close(fd) + return True + except OSError: + return False + finally: + page.close() + with contextlib.suppress(OSError): + os.remove(path) + + def _ensure_dirs(path: str) -> None: """Create parent directories of *path* if they don't exist.""" os.makedirs(os.path.dirname(path), exist_ok=True) @@ -34,6 +64,7 @@ def store_block( buffer: memoryview, offset: int, block_size: int, + use_o_direct: bool = True, ) -> None: """ Store callback: Writes to a temp file then atomically replaces the destination. @@ -49,10 +80,11 @@ def store_block( # Write block atomically. Cast to a flat byte view so the slice uses byte # indices; the raw memoryview may be multi-dimensional with itemsize > 1. view_slice = buffer.cast("B")[offset : offset + block_size] + o_direct = O_DIRECT if use_o_direct else 0 try: fd = os.open( tmp_path, - os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC | O_DIRECT, + os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC | o_direct, 0o644, ) try: @@ -77,14 +109,16 @@ def load_block( view: memoryview, offset: int, block_size: int, + use_o_direct: bool = True, ) -> None: """ Load callback: read one KV block from disk. Remove the file on failure. """ fd: int | None = None view_slice = view.cast("B")[offset : offset + block_size] + o_direct = O_DIRECT if use_o_direct else 0 try: - fd = os.open(source_path, os.O_RDONLY | O_DIRECT) + fd = os.open(source_path, os.O_RDONLY | o_direct) bytes_read = os.readv(fd, [view_slice]) if bytes_read < block_size: raise OSError(f"Short read: expected {block_size} bytes, read {bytes_read}") diff --git a/vllm/v1/kv_offload/tiering/fs/manager.py b/vllm/v1/kv_offload/tiering/fs/manager.py index f12ef2c6d4..4fde24d760 100644 --- a/vllm/v1/kv_offload/tiering/fs/manager.py +++ b/vllm/v1/kv_offload/tiering/fs/manager.py @@ -49,7 +49,7 @@ from vllm.v1.kv_offload.tiering.base import ( ScheduleEndContext, SecondaryTierManager, ) -from vllm.v1.kv_offload.tiering.fs.io import load_block, store_block +from vllm.v1.kv_offload.tiering.fs.io import load_block, probe_o_direct, store_block from vllm.v1.kv_offload.tiering.fs.thread_pool import DualQueueThreadPool if TYPE_CHECKING: @@ -168,6 +168,18 @@ class FileSystemTierManager(SecondaryTierManager): self.file_mapper.get_run_config(), f, indent=2, sort_keys=True ) + # Prefer O_DIRECT to bypass the page cache, but fall back to buffered + # I/O on filesystems that reject it (e.g. overlayfs, some NFS mounts) + # rather than failing every block. + self._use_o_direct = probe_o_direct(os.path.dirname(config_path)) + if not self._use_o_direct: + logger.warning( + "O_DIRECT is not supported at '%s'; falling back to buffered " + "I/O for the '%s' KV offload tier.", + root_dir, + tier_type, + ) + self._pool = DualQueueThreadPool( n_read_threads, n_write_threads, @@ -198,6 +210,7 @@ class FileSystemTierManager(SecondaryTierManager): self._primary_kv_view, int(bid) * self._block_size, self._block_size, + self._use_o_direct, ) for key, bid in zip(job_metadata.keys, job_metadata.block_ids) ) @@ -212,6 +225,7 @@ class FileSystemTierManager(SecondaryTierManager): self._primary_kv_view, int(bid) * self._block_size, self._block_size, + self._use_o_direct, ) for key, bid in zip(job_metadata.keys, job_metadata.block_ids) )