diff --git a/csrc/fs_io.cpp b/csrc/fs_io.cpp index fdf3e614e6..6b7c8d7fe1 100644 --- a/csrc/fs_io.cpp +++ b/csrc/fs_io.cpp @@ -3,19 +3,155 @@ #include +#include +#include #include +#include +#include #include +#if defined(O_DIRECT) +constexpr int kODirectFlag = O_DIRECT; +#else +constexpr int kODirectFlag = 0; +#endif + extern "C" { -static void _batch_lookup(const std::vector& paths, +namespace { + +// Returns 0 on success, or the std::error_code's POSIX-compatible value on +// failure, mirroring the errno convention used by the syscalls below. +inline int ensure_parent_dirs(const std::string& path) { + const auto parent = std::filesystem::path(path).parent_path(); + if (parent.empty()) { + return 0; + } + std::error_code ec; + std::filesystem::create_directories(parent, ec); + return ec ? ec.value() : 0; +} + +// Core single-block store: src/size are raw pointer + byte count. Returns 0 +// on success, or the errno of the failing step on failure -- captured +// before any subsequent cleanup call can overwrite it. On failure, the temp +// file is removed. +inline int _store_block(const char* tmp_path, const char* dest_path, + const char* src, size_t size, bool use_o_direct) { + if (access(dest_path, F_OK) == 0) { + return 0; // Already present. + } + + if (const int err = ensure_parent_dirs(dest_path); err != 0) { + return err; + } + + const int o_direct_flag = use_o_direct ? kODirectFlag : 0; + const int fd = open( + tmp_path, O_CREAT | O_EXCL | O_WRONLY | O_TRUNC | o_direct_flag, 0644); + if (fd < 0) { + return errno; + } + + const ssize_t written = write(fd, src, size); + if (written < 0 || static_cast(written) != size) { + const int err = written < 0 ? errno : EIO; + close(fd); // Best-effort cleanup; the real error is already captured. + unlink(tmp_path); + return err; + } + + if (close(fd) != 0) { + const int err = errno; + unlink(tmp_path); + return err; + } + + if (rename(tmp_path, dest_path) != 0) { + const int err = errno; + unlink(tmp_path); + return err; + } + + return 0; +} + +// Core single-block load: dst/size are raw pointer + byte count. Returns 0 +// on success, or the errno of the failing step on failure. On failure, +// the source file is removed since a partially-read block should not be reused. +inline int _load_block(const char* source_path, char* dst, size_t size, + bool use_o_direct) { + const int o_direct_flag = use_o_direct ? kODirectFlag : 0; + const int fd = open(source_path, O_RDONLY | o_direct_flag, 0); + if (fd < 0) { + const int err = errno; + unlink(source_path); + return err; + } + + const ssize_t bytes_read = read(fd, dst, size); + if (bytes_read < 0 || static_cast(bytes_read) != size) { + const int err = bytes_read < 0 ? errno : EIO; + close(fd); + unlink(source_path); + return err; + } + + if (close(fd) != 0) { + const int err = errno; + unlink(source_path); + return err; + } + + return 0; +} + +inline void _batch_lookup(const std::vector& paths, std::vector& exists_flags) { for (size_t i = 0; i < paths.size(); i++) { exists_flags[i] = (access(paths[i], F_OK) == 0) ? 1 : 0; } } +// Helper: extract a list[str] of length n into a vector. +// Returns false and sets a Python exception on error. +inline bool extract_str_list(PyObject* list, Py_ssize_t n, + std::vector& out) { + for (Py_ssize_t i = 0; i < n; i++) { + out[i] = PyUnicode_AsUTF8AndSize(PyList_GetItem(list, i), nullptr); + if (out[i] == nullptr) { + return false; + } + } + return true; +} + +// Helper: extract a Py_buffer per element of a list[bytes-like] of length n. +// On success, `out` holds n acquired buffers (caller must PyBuffer_Release +// each). On failure, any buffers already acquired are released before +// returning false, and a Python exception is set. +inline bool extract_buffer_list(PyObject* list, Py_ssize_t n, int flags, + std::vector& out) { + for (Py_ssize_t i = 0; i < n; i++) { + if (PyObject_GetBuffer(PyList_GetItem(list, i), &out[i], flags) != 0) { + for (Py_ssize_t j = 0; j < i; j++) { + PyBuffer_Release(&out[j]); + } + return false; + } + } + return true; +} + +inline void release_buffer_list(std::vector& buffers) { + for (auto& buf : buffers) { + PyBuffer_Release(&buf); + } +} + +} // namespace + /// @brief Check file existence for a batch of paths. /// @param paths list[str] – absolute paths to check. /// @return list[bool] – True if the corresponding path exists, False otherwise. @@ -51,11 +187,157 @@ static PyObject* batch_lookup(PyObject* /*self*/, PyObject* args) { return result; } +/// @brief Store a batch of blocks, each from its own buffer, to disk. +/// @param tmp_paths list[str] – one temp path per block. +/// @param dest_paths list[str] – one destination path per block. +/// @param buffers list[bytes-like] – one source buffer per block. +/// @param use_o_direct bool – whether to open files with O_DIRECT +/// (default True). Ignored where O_DIRECT is unsupported +/// by the platform. +/// @note Releases the GIL for the entire batch. Raises on first error. +static PyObject* batch_store_block(PyObject* /*self*/, PyObject* args) { + PyObject* tmp_paths_obj = nullptr; + PyObject* dest_paths_obj = nullptr; + PyObject* buffers_obj = nullptr; + int use_o_direct = 1; + + if (!PyArg_ParseTuple(args, "O!O!O!|p", &PyList_Type, &tmp_paths_obj, + &PyList_Type, &dest_paths_obj, &PyList_Type, + &buffers_obj, &use_o_direct)) { + return nullptr; + } + + const Py_ssize_t n = PyList_Size(tmp_paths_obj); + if (PyList_Size(dest_paths_obj) != n || PyList_Size(buffers_obj) != n) { + PyErr_SetString( + PyExc_ValueError, + "tmp_paths, dest_paths and buffers must have the same length"); + return nullptr; + } + + std::vector tmp_paths(n); + std::vector dest_paths(n); + + if (!extract_str_list(tmp_paths_obj, n, tmp_paths)) return nullptr; + if (!extract_str_list(dest_paths_obj, n, dest_paths)) return nullptr; + + std::vector buffers(n); + if (!extract_buffer_list(buffers_obj, n, PyBUF_SIMPLE, buffers)) { + return nullptr; + } + + Py_ssize_t failed_index = -1; + int failure_errno = 0; + + { + Py_BEGIN_ALLOW_THREADS for (Py_ssize_t i = 0; i < n; i++) { + const char* buf = static_cast(buffers[i].buf); + const int err = + _store_block(tmp_paths[i], dest_paths[i], buf, + static_cast(buffers[i].len), use_o_direct); + if (err != 0) { + failed_index = i; + failure_errno = err; + break; + } + } + Py_END_ALLOW_THREADS + } + + release_buffer_list(buffers); + + if (failed_index >= 0) { + // PyErr_SetFromErrnoWithFilename() reads the errno to format exception. + errno = failure_errno; + return PyErr_SetFromErrnoWithFilename(PyExc_OSError, + dest_paths[failed_index]); + } + + Py_RETURN_NONE; +} + +/// @brief Load a batch of blocks from disk, each into its own buffer. +/// @param source_paths list[str] – one source path per block. +/// @param buffers list[writable bytes-like] – one destination buffer +/// per block. +/// @param use_o_direct bool – whether to open files with O_DIRECT +/// (default True). Ignored where O_DIRECT is unsupported +/// by the platform. +/// @note Releases the GIL for the entire batch. Raises on first error. +static PyObject* batch_load_block(PyObject* /*self*/, PyObject* args) { + PyObject* source_paths_obj = nullptr; + PyObject* buffers_obj = nullptr; + int use_o_direct = 1; + + if (!PyArg_ParseTuple(args, "O!O!|p", &PyList_Type, &source_paths_obj, + &PyList_Type, &buffers_obj, &use_o_direct)) { + return nullptr; + } + + const Py_ssize_t n = PyList_Size(source_paths_obj); + if (PyList_Size(buffers_obj) != n) { + PyErr_SetString(PyExc_ValueError, + "source_paths and buffers must have the same length"); + return nullptr; + } + + std::vector source_paths(n); + if (!extract_str_list(source_paths_obj, n, source_paths)) return nullptr; + + std::vector buffers(n); + if (!extract_buffer_list(buffers_obj, n, PyBUF_WRITABLE, buffers)) { + return nullptr; + } + + Py_ssize_t failed_index = -1; + int failure_errno = 0; + + { + Py_BEGIN_ALLOW_THREADS for (Py_ssize_t i = 0; i < n; i++) { + char* buf = static_cast(buffers[i].buf); + const int err = + _load_block(source_paths[i], buf, static_cast(buffers[i].len), + use_o_direct); + if (err != 0) { + failed_index = i; + failure_errno = err; + break; + } + } + Py_END_ALLOW_THREADS + } + + release_buffer_list(buffers); + + if (failed_index >= 0) { + // PyErr_SetFromErrnoWithFilename() reads the errno to format exception. + errno = failure_errno; + return PyErr_SetFromErrnoWithFilename(PyExc_OSError, + source_paths[failed_index]); + } + + Py_RETURN_NONE; +} + static PyMethodDef fs_io_C_methods[] = { {"batch_lookup", batch_lookup, METH_VARARGS, "batch_lookup(paths: list[str]) -> list[bool]\n" "\n" "Check file existence for a batch of paths."}, + {"batch_store_block", batch_store_block, METH_VARARGS, + "batch_store_block(tmp_paths: list[str], dest_paths: list[str],\n" + " buffers: list[bytes-like],\n" + " use_o_direct: bool = True) -> None\n" + "\n" + "Store a batch of blocks, each from its own buffer, to disk. Raises on " + "first error."}, + {"batch_load_block", batch_load_block, METH_VARARGS, + "batch_load_block(source_paths: list[str],\n" + " buffers: list[writable bytes-like],\n" + " use_o_direct: bool = True) -> None\n" + "\n" + "Load a batch of blocks from disk into corresponding buffers. " + "Raises on first error."}, {nullptr, nullptr, 0, nullptr}, }; diff --git a/tests/v1/kv_offload/tiering/test_fs_tier.py b/tests/v1/kv_offload/tiering/test_fs_tier.py index 68b5512a3e..66533e3676 100644 --- a/tests/v1/kv_offload/tiering/test_fs_tier.py +++ b/tests/v1/kv_offload/tiering/test_fs_tier.py @@ -46,6 +46,7 @@ from vllm.v1.kv_offload.tiering.fs.thread_pool import DualQueueThreadPool # Helpers # --------------------------------------------------------------------------- +_NUM_BLOCKS = 8 _BLOCK_ELEMENTS = 128 * mmap.PAGESIZE # 2MB per block for pagesize 4096. _DTYPE: torch.dtype = torch.float32 _CTX = ReqContext(req_id="test") @@ -162,7 +163,7 @@ def _page_aligned_rand_tensor( @pytest.fixture def fs_tier(tmp_path): - tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS) + tensor = _page_aligned_zero_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS) mock_view = memoryview(tensor.numpy()) tier = FileSystemTierManager( offloading_spec=_MOCK_OFFLOADING_SPEC, @@ -178,7 +179,7 @@ def fs_tier(tmp_path): @pytest.fixture def fs_tier_with_events(tmp_path): - tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS) + tensor = _page_aligned_zero_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS) mock_view = memoryview(tensor.numpy()) tier = FileSystemTierManager( offloading_spec=_make_offloading_spec(enable_kv_cache_events=True), @@ -347,33 +348,43 @@ def test_shutdown_discards_pending_tasks(fs_tier): assert all(not t.is_alive() for t in tier._pool._threads) -def test_store_load_data_integrity(fs_tier): - """Data written by store must be exactly recovered by load.""" +@pytest.mark.parametrize("batch_size", [0, 1, 2, 5]) +@pytest.mark.parametrize("use_c_ext", [True, False]) +def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size): + """Data written by store must be exactly recovered by load, for batches + of any size -- including the empty batch.""" + import vllm.v1.kv_offload.tiering.fs.io as io_mod + + if use_c_ext and not io_mod._HAS_FSIO_C: + pytest.skip("fs_io_C extension not built") + monkeypatch.setattr(io_mod, "_HAS_FSIO_C", use_c_ext) + tier, tensor = fs_tier # Populate tensor with random data - tensor[:] = _page_aligned_rand_tensor(4, _BLOCK_ELEMENTS) + tensor[:] = _page_aligned_rand_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS) - # Store first 2 blocks - num_store = 2 - expected = tensor[:num_store].clone() + keys = [key(i) for i in range(batch_size)] + store_block_ids = list(range(batch_size)) + load_block_ids = list(range(_NUM_BLOCKS - batch_size, _NUM_BLOCKS)) + expected = tensor[:batch_size].clone() - store_ids = list(range(num_store)) - keys = [key(i) for i in range(num_store)] + tier.submit_store(make_job(1, keys, store_block_ids)) + store_results = drain(tier) + assert len(store_results) == 1 + assert store_results[0].success + assert all(os.path.exists(tier.file_mapper.get_file_name(k)) for k in keys) - tier.submit_store(make_job(1, keys, store_ids)) - results = drain(tier) - assert all(r.success for r in results) + # reset tensor to prove data is read from disk + tensor[:] = 0.0 - # Overwrite source blocks to prove data is read from disk - tensor[:num_store] = 0.0 + # Load into a range disjoint by index from the store ids, to also + # exercise loading a block into a different id than it was stored from. + tier.submit_load(make_job(2, keys, load_block_ids, is_promotion=True)) + load_results = drain(tier) + assert len(load_results) == 1 + assert load_results[0].success - # Load into last 2 blocks - load_ids = [2, 3] - tier.submit_load(make_job(2, keys, load_ids, is_promotion=True)) - results = drain(tier) - assert all(r.success for r in results) - - for i, bid in enumerate(load_ids): + for i, bid in enumerate(load_block_ids): assert torch.allclose(tensor[bid], expected[i]), ( f"Block {bid} data mismatch after store+load" ) @@ -499,6 +510,30 @@ def test_batch_lookup_dispatch(fs_tier, monkeypatch, use_c_ext): assert results == [LookupResult.HIT, LookupResult.MISS] +@pytest.mark.parametrize("use_c_ext", [True, False]) +def test_out_of_bounds_block_id_smoke(fs_tier, monkeypatch, use_c_ext): + """Smoke test: a block id beyond the primary tensor's block count must + fail the job, for both the C extension and the Python fallback.""" + import vllm.v1.kv_offload.tiering.fs.io as io_mod + + if use_c_ext and not io_mod._HAS_FSIO_C: + pytest.skip("fs_io_C extension not built") + monkeypatch.setattr(io_mod, "_HAS_FSIO_C", use_c_ext) + + tier, tensor = fs_tier + out_of_bounds_bid = tensor.shape[0] # one past the last valid block + + tier.submit_store(make_job(1, [key(1)], [out_of_bounds_bid])) + store_results = drain(tier) + assert len(store_results) == 1 + assert not store_results[0].success + + tier.submit_load(make_job(2, [key(1)], [out_of_bounds_bid], is_promotion=True)) + load_results = drain(tier) + assert len(load_results) == 1 + assert not load_results[0].success + + # --------------------------------------------------------------------------- # KV events # --------------------------------------------------------------------------- @@ -571,14 +606,14 @@ def test_mixed_job_results_emit_event_only_for_successful_job( tier = fs_tier_with_events failing_path = tier.file_mapper.get_file_name(key(1)) - original_store_block = mgr_mod.store_block + original_batch_store_block = mgr_mod.batch_store_block - def flaky_store_block(dest_path, *args, **kwargs): - if dest_path == failing_path: + def flaky_batch_store_block(paths, *args, **kwargs): + if failing_path in paths: raise OSError("injected store failure") - return original_store_block(dest_path, *args, **kwargs) + return original_batch_store_block(paths, *args, **kwargs) - monkeypatch.setattr(mgr_mod, "store_block", flaky_store_block) + monkeypatch.setattr(mgr_mod, "batch_store_block", flaky_batch_store_block) tier.submit_store(make_job(1, [key(1)], [0])) tier.submit_store(make_job(2, [key(2)], [1])) @@ -599,14 +634,14 @@ def test_partially_failed_store_emits_no_event(fs_tier_with_events, monkeypatch) tier = fs_tier_with_events failing_path = tier.file_mapper.get_file_name(key(2)) - original_store_block = mgr_mod.store_block + original_batch_store_block = mgr_mod.batch_store_block - def flaky_store_block(dest_path, *args, **kwargs): - if dest_path == failing_path: + def flaky_batch_store_block(paths, *args, **kwargs): + if failing_path in paths: raise OSError("injected store failure") - return original_store_block(dest_path, *args, **kwargs) + return original_batch_store_block(paths, *args, **kwargs) - monkeypatch.setattr(mgr_mod, "store_block", flaky_store_block) + monkeypatch.setattr(mgr_mod, "batch_store_block", flaky_batch_store_block) tier.submit_store(make_job(1, [key(1), key(2)], [0, 1])) results = drain(tier) diff --git a/vllm/v1/kv_offload/tiering/fs/io.py b/vllm/v1/kv_offload/tiering/fs/io.py index 7785591fd2..7c70fbae9f 100644 --- a/vllm/v1/kv_offload/tiering/fs/io.py +++ b/vllm/v1/kv_offload/tiering/fs/io.py @@ -8,6 +8,18 @@ import os import random import threading +try: + from vllm.fs_io_C import ( # pyright: ignore[reportMissingImports] + batch_load_block as batch_load_block_C, + ) + from vllm.fs_io_C import ( + batch_store_block as batch_store_block_C, + ) + + _HAS_FSIO_C = True +except ImportError: + _HAS_FSIO_C = False + logger = logging.getLogger(__name__) # O_DIRECT is Linux-specific and not available on macOS @@ -59,7 +71,23 @@ def _ensure_dirs(path: str) -> None: os.makedirs(os.path.dirname(path), exist_ok=True) -def store_block( +def _validate_offsets(view: memoryview, offsets: list[int], block_size: int) -> None: + """Raise if any block would read/write past the bounds of `view`. + + Without this, an out-of-range offset silently clips to a shorter (or + empty) slice instead of failing, since memoryview slicing follows + Python's slice-clamping semantics rather than raising. + """ + total_len = len(view.cast("B")) + for offset in offsets: + if offset < 0 or offset + block_size > total_len: + raise ValueError( + f"block offset {offset} (block_size {block_size}) is out of " + f"bounds for a buffer of size {total_len}" + ) + + +def _store_block( dest_path: str, buffer: memoryview, offset: int, @@ -104,7 +132,7 @@ def store_block( raise -def load_block( +def _load_block( source_path: str, view: memoryview, offset: int, @@ -117,6 +145,7 @@ def load_block( fd: int | None = None view_slice = view.cast("B")[offset : offset + block_size] o_direct = O_DIRECT if use_o_direct else 0 + try: fd = os.open(source_path, os.O_RDONLY | o_direct) bytes_read = os.readv(fd, [view_slice]) @@ -133,3 +162,52 @@ def load_block( finally: if fd is not None: os.close(fd) + + +def batch_store_block( + paths: list[str], + view: memoryview, + offsets: list[int], + block_size: int, + use_o_direct: bool = True, +) -> None: + """ + Store a batch of KV blocks from a shared buffer to disk in one call. + + Each block buffer[offsets[i] : offsets[i]+block_size] is written atomically + to dest_paths[i] via a temp-file rename. Raises on first error. + """ + _validate_offsets(view, offsets, block_size) + + if _HAS_FSIO_C: + view_B = view.cast("B") + view_slices = [view_B[x : x + block_size] for x in offsets] + tmp_paths = [p + _get_tmp_suffix() for p in paths] + return batch_store_block_C(tmp_paths, paths, view_slices, use_o_direct) + else: + for path, offset in zip(paths, offsets): + _store_block(path, view, offset, block_size, use_o_direct) + + +def batch_load_block( + paths: list[str], + view: memoryview, + offsets: list[int], + block_size: int, + use_o_direct: bool = True, +) -> None: + """ + Load a batch of KV blocks from disk into a shared buffer in one call. + + Block i is read from source_paths[i] into view[offsets[i] : offsets[i]+block_size]. + Raises on first error and removes the offending file. + """ + _validate_offsets(view, offsets, block_size) + + if _HAS_FSIO_C: + view_B = view.cast("B") + view_slices = [view_B[x : x + block_size] for x in offsets] + return batch_load_block_C(paths, view_slices, use_o_direct) + else: + for path, offset in zip(paths, offsets): + _load_block(path, view, offset, block_size, use_o_direct) diff --git a/vllm/v1/kv_offload/tiering/fs/manager.py b/vllm/v1/kv_offload/tiering/fs/manager.py index 4fe200629f..1c8d8b2667 100644 --- a/vllm/v1/kv_offload/tiering/fs/manager.py +++ b/vllm/v1/kv_offload/tiering/fs/manager.py @@ -49,7 +49,11 @@ from vllm.v1.kv_offload.tiering.base import ( ScheduleEndContext, SecondaryTierManager, ) -from vllm.v1.kv_offload.tiering.fs.io import load_block, probe_o_direct, store_block +from vllm.v1.kv_offload.tiering.fs.io import ( + batch_load_block, + batch_store_block, + probe_o_direct, +) from vllm.v1.kv_offload.tiering.fs.thread_pool import DualQueueThreadPool if TYPE_CHECKING: @@ -203,33 +207,28 @@ class FileSystemTierManager(SecondaryTierManager): def submit_store(self, job_metadata: JobMetadata) -> None: if self.events is not None: self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys) - tasks = ( - functools.partial( - store_block, - self.file_mapper.get_file_name(key), - self._primary_kv_view, - int(bid) * self._block_size, - self._block_size, - self._use_o_direct, - ) - for key, bid in zip(job_metadata.keys, job_metadata.block_ids) + task = functools.partial( + batch_store_block, + [self.file_mapper.get_file_name(key) for key in job_metadata.keys], + self._primary_kv_view, + [int(bid) * self._block_size for bid in job_metadata.block_ids], + self._block_size, + self._use_o_direct, ) - self._pool.enqueue_store(job_metadata.job_id, len(job_metadata.keys), tasks) + self._pool.enqueue_store(job_metadata.job_id, 1, [task]) @override def submit_load(self, job_metadata: JobMetadata) -> None: - tasks = ( - functools.partial( - load_block, - self.file_mapper.get_file_name(key), - self._primary_kv_view, - int(bid) * self._block_size, - self._block_size, - self._use_o_direct, - ) - for key, bid in zip(job_metadata.keys, job_metadata.block_ids) + task = functools.partial( + batch_load_block, + [self.file_mapper.get_file_name(key) for key in job_metadata.keys], + self._primary_kv_view, + [int(bid) * self._block_size for bid in job_metadata.block_ids], + self._block_size, + self._use_o_direct, ) - self._pool.enqueue_load(job_metadata.job_id, len(job_metadata.keys), tasks) + + self._pool.enqueue_load(job_metadata.job_id, 1, [task]) @override def get_finished_jobs(self) -> Iterable[JobResult]: