[KV Offload][CI] Fall back to buffered I/O without O_DIRECT; fix flaky api-server test (#49734)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Spawn worker targets kept free of heavy imports.
|
||||
|
||||
``multiprocessing`` with the ``spawn`` start method re-imports the module that
|
||||
defines a process target in the child. Housing these stubs in a stdlib-only
|
||||
module keeps child startup fast and deterministic, instead of paying a multi-
|
||||
second ``import vllm`` before the child can run.
|
||||
"""
|
||||
|
||||
|
||||
def exit_before_report_worker(listen_address, sock, args, client_config=None):
|
||||
"""Exit immediately without touching ``actual_address_pipe``."""
|
||||
return
|
||||
@@ -10,6 +10,9 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
import zmq
|
||||
|
||||
from tests.entrypoints.unit_tests._api_server_spawn_workers import (
|
||||
exit_before_report_worker,
|
||||
)
|
||||
from vllm.utils.network_utils import make_zmq_socket, split_zmq_path
|
||||
from vllm.v1.utils import (
|
||||
APIServerProcessManager,
|
||||
@@ -228,6 +231,7 @@ def test_normal_completion(api_server_args):
|
||||
def test_external_process_monitoring(api_server_args):
|
||||
"""Test that wait_for_completion_or_failure handles additional processes."""
|
||||
global WORKER_RUNTIME_SECONDS
|
||||
prev_worker_runtime = WORKER_RUNTIME_SECONDS
|
||||
WORKER_RUNTIME_SECONDS = 100
|
||||
|
||||
# Create and start the external process
|
||||
@@ -307,6 +311,7 @@ def test_external_process_monitoring(api_server_args):
|
||||
manager.shutdown()
|
||||
mock_coordinator.shutdown()
|
||||
time.sleep(0.2)
|
||||
WORKER_RUNTIME_SECONDS = prev_worker_runtime
|
||||
|
||||
|
||||
@pytest.mark.timeout(60)
|
||||
@@ -383,10 +388,10 @@ def test_gather_actual_addresses_child_crash_before_report():
|
||||
num_servers=num_servers,
|
||||
input_addresses=placeholder_inputs,
|
||||
output_addresses=placeholder_outputs,
|
||||
# mock_run_api_server_worker exits without touching
|
||||
# exit_before_report_worker exits without touching
|
||||
# ``actual_address_pipe`` — simulates a child that dies before
|
||||
# reporting its bound addresses.
|
||||
target_server_fn=mock_run_api_server_worker,
|
||||
target_server_fn=exit_before_report_worker,
|
||||
)
|
||||
try:
|
||||
# Sentinel-first vs pipe-EOF-first both produce "reporting".
|
||||
|
||||
@@ -379,6 +379,43 @@ def test_store_load_data_integrity(fs_tier):
|
||||
)
|
||||
|
||||
|
||||
def test_store_load_roundtrip_without_o_direct(tmp_path, monkeypatch):
|
||||
"""Buffered fallback must round-trip data when O_DIRECT is unsupported.
|
||||
|
||||
Simulates filesystems (e.g. overlayfs, some NFS) that reject O_DIRECT by
|
||||
forcing the capability probe to report it unavailable.
|
||||
"""
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.kv_offload.tiering.fs.manager.probe_o_direct",
|
||||
lambda _dir: False,
|
||||
)
|
||||
tensor = _page_aligned_rand_tensor(4, _BLOCK_ELEMENTS)
|
||||
tier = FileSystemTierManager(
|
||||
offloading_spec=_MOCK_OFFLOADING_SPEC,
|
||||
primary_kv_view=memoryview(tensor.numpy()),
|
||||
tier_type="fs",
|
||||
root_dir=str(tmp_path),
|
||||
n_read_threads=4,
|
||||
n_write_threads=4,
|
||||
)
|
||||
try:
|
||||
assert tier._use_o_direct is False
|
||||
|
||||
keys = [key(0), key(1)]
|
||||
expected = tensor[:2].clone()
|
||||
tier.submit_store(make_job(1, keys, [0, 1]))
|
||||
assert all(r.success for r in drain(tier))
|
||||
|
||||
tensor[:2] = 0.0
|
||||
tier.submit_load(make_job(2, keys, [2, 3], is_promotion=True))
|
||||
assert all(r.success for r in drain(tier))
|
||||
|
||||
for i, bid in enumerate([2, 3]):
|
||||
assert torch.allclose(tensor[bid], expected[i])
|
||||
finally:
|
||||
tier.shutdown()
|
||||
|
||||
|
||||
def test_wait_idle_blocks_until_tasks_complete():
|
||||
"""wait_idle must not return while a task is still in flight."""
|
||||
pool = DualQueueThreadPool(n_read_threads=1, n_write_threads=1)
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import mmap
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
@@ -24,6 +26,34 @@ def _get_tmp_suffix() -> str:
|
||||
return _thread_local.tmp_suffix
|
||||
|
||||
|
||||
def probe_o_direct(directory: str) -> bool:
|
||||
"""Return whether ``O_DIRECT`` I/O works in *directory*.
|
||||
|
||||
``O_DIRECT`` is unsupported on some filesystems (e.g. the overlayfs backing
|
||||
a container ``/tmp``, older tmpfs, or some NFS mounts), where opening or
|
||||
writing a file with it fails with ``EINVAL``. Probe once with an aligned
|
||||
single-page write so callers can fall back to buffered I/O instead of
|
||||
failing on every block.
|
||||
"""
|
||||
if not O_DIRECT:
|
||||
return False
|
||||
path = os.path.join(directory, f".o_direct_probe{_get_tmp_suffix()}")
|
||||
page = mmap.mmap(-1, mmap.PAGESIZE)
|
||||
try:
|
||||
fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC | O_DIRECT, 0o644)
|
||||
try:
|
||||
os.write(fd, page)
|
||||
finally:
|
||||
os.close(fd)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
finally:
|
||||
page.close()
|
||||
with contextlib.suppress(OSError):
|
||||
os.remove(path)
|
||||
|
||||
|
||||
def _ensure_dirs(path: str) -> None:
|
||||
"""Create parent directories of *path* if they don't exist."""
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
@@ -34,6 +64,7 @@ def store_block(
|
||||
buffer: memoryview,
|
||||
offset: int,
|
||||
block_size: int,
|
||||
use_o_direct: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Store callback: Writes to a temp file then atomically replaces the destination.
|
||||
@@ -49,10 +80,11 @@ def store_block(
|
||||
# Write block atomically. Cast to a flat byte view so the slice uses byte
|
||||
# indices; the raw memoryview may be multi-dimensional with itemsize > 1.
|
||||
view_slice = buffer.cast("B")[offset : offset + block_size]
|
||||
o_direct = O_DIRECT if use_o_direct else 0
|
||||
try:
|
||||
fd = os.open(
|
||||
tmp_path,
|
||||
os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC | O_DIRECT,
|
||||
os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC | o_direct,
|
||||
0o644,
|
||||
)
|
||||
try:
|
||||
@@ -77,14 +109,16 @@ def load_block(
|
||||
view: memoryview,
|
||||
offset: int,
|
||||
block_size: int,
|
||||
use_o_direct: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Load callback: read one KV block from disk. Remove the file on failure.
|
||||
"""
|
||||
fd: int | None = None
|
||||
view_slice = view.cast("B")[offset : offset + block_size]
|
||||
o_direct = O_DIRECT if use_o_direct else 0
|
||||
try:
|
||||
fd = os.open(source_path, os.O_RDONLY | O_DIRECT)
|
||||
fd = os.open(source_path, os.O_RDONLY | o_direct)
|
||||
bytes_read = os.readv(fd, [view_slice])
|
||||
if bytes_read < block_size:
|
||||
raise OSError(f"Short read: expected {block_size} bytes, read {bytes_read}")
|
||||
|
||||
@@ -49,7 +49,7 @@ from vllm.v1.kv_offload.tiering.base import (
|
||||
ScheduleEndContext,
|
||||
SecondaryTierManager,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.fs.io import load_block, store_block
|
||||
from vllm.v1.kv_offload.tiering.fs.io import load_block, probe_o_direct, store_block
|
||||
from vllm.v1.kv_offload.tiering.fs.thread_pool import DualQueueThreadPool
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -168,6 +168,18 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
self.file_mapper.get_run_config(), f, indent=2, sort_keys=True
|
||||
)
|
||||
|
||||
# Prefer O_DIRECT to bypass the page cache, but fall back to buffered
|
||||
# I/O on filesystems that reject it (e.g. overlayfs, some NFS mounts)
|
||||
# rather than failing every block.
|
||||
self._use_o_direct = probe_o_direct(os.path.dirname(config_path))
|
||||
if not self._use_o_direct:
|
||||
logger.warning(
|
||||
"O_DIRECT is not supported at '%s'; falling back to buffered "
|
||||
"I/O for the '%s' KV offload tier.",
|
||||
root_dir,
|
||||
tier_type,
|
||||
)
|
||||
|
||||
self._pool = DualQueueThreadPool(
|
||||
n_read_threads,
|
||||
n_write_threads,
|
||||
@@ -198,6 +210,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
self._primary_kv_view,
|
||||
int(bid) * self._block_size,
|
||||
self._block_size,
|
||||
self._use_o_direct,
|
||||
)
|
||||
for key, bid in zip(job_metadata.keys, job_metadata.block_ids)
|
||||
)
|
||||
@@ -212,6 +225,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
self._primary_kv_view,
|
||||
int(bid) * self._block_size,
|
||||
self._block_size,
|
||||
self._use_o_direct,
|
||||
)
|
||||
for key, bid in zip(job_metadata.keys, job_metadata.block_ids)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user