[KV-offload][FS] : Batch store/load_block in C (#49152)

Signed-off-by: <>
Co-authored-by: Varun Sundar Rabindranath <varun-sundar-rabindranath@h100-01.nemg-001.lab.rdu2.dc.redhat.com>
This commit is contained in:
Varun Sundar Rabindranath
2026-07-28 11:21:53 +08:00
committed by GitHub
co-authored by Varun Sundar Rabindranath
parent 60915c972c
commit d18ed2304a
4 changed files with 452 additions and 58 deletions
+283 -1
View File
@@ -3,19 +3,155 @@
#include <Python.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
#include <filesystem>
#include <string>
#include <vector>
#if defined(O_DIRECT)
constexpr int kODirectFlag = O_DIRECT;
#else
constexpr int kODirectFlag = 0;
#endif
extern "C" {
static void _batch_lookup(const std::vector<const char*>& paths,
namespace {
// Returns 0 on success, or the std::error_code's POSIX-compatible value on
// failure, mirroring the errno convention used by the syscalls below.
inline int ensure_parent_dirs(const std::string& path) {
const auto parent = std::filesystem::path(path).parent_path();
if (parent.empty()) {
return 0;
}
std::error_code ec;
std::filesystem::create_directories(parent, ec);
return ec ? ec.value() : 0;
}
// Core single-block store: src/size are raw pointer + byte count. Returns 0
// on success, or the errno of the failing step on failure -- captured
// before any subsequent cleanup call can overwrite it. On failure, the temp
// file is removed.
inline int _store_block(const char* tmp_path, const char* dest_path,
const char* src, size_t size, bool use_o_direct) {
if (access(dest_path, F_OK) == 0) {
return 0; // Already present.
}
if (const int err = ensure_parent_dirs(dest_path); err != 0) {
return err;
}
const int o_direct_flag = use_o_direct ? kODirectFlag : 0;
const int fd = open(
tmp_path, O_CREAT | O_EXCL | O_WRONLY | O_TRUNC | o_direct_flag, 0644);
if (fd < 0) {
return errno;
}
const ssize_t written = write(fd, src, size);
if (written < 0 || static_cast<size_t>(written) != size) {
const int err = written < 0 ? errno : EIO;
close(fd); // Best-effort cleanup; the real error is already captured.
unlink(tmp_path);
return err;
}
if (close(fd) != 0) {
const int err = errno;
unlink(tmp_path);
return err;
}
if (rename(tmp_path, dest_path) != 0) {
const int err = errno;
unlink(tmp_path);
return err;
}
return 0;
}
// Core single-block load: dst/size are raw pointer + byte count. Returns 0
// on success, or the errno of the failing step on failure. On failure,
// the source file is removed since a partially-read block should not be reused.
inline int _load_block(const char* source_path, char* dst, size_t size,
bool use_o_direct) {
const int o_direct_flag = use_o_direct ? kODirectFlag : 0;
const int fd = open(source_path, O_RDONLY | o_direct_flag, 0);
if (fd < 0) {
const int err = errno;
unlink(source_path);
return err;
}
const ssize_t bytes_read = read(fd, dst, size);
if (bytes_read < 0 || static_cast<size_t>(bytes_read) != size) {
const int err = bytes_read < 0 ? errno : EIO;
close(fd);
unlink(source_path);
return err;
}
if (close(fd) != 0) {
const int err = errno;
unlink(source_path);
return err;
}
return 0;
}
inline void _batch_lookup(const std::vector<const char*>& paths,
std::vector<int>& exists_flags) {
for (size_t i = 0; i < paths.size(); i++) {
exists_flags[i] = (access(paths[i], F_OK) == 0) ? 1 : 0;
}
}
// Helper: extract a list[str] of length n into a vector<const char*>.
// Returns false and sets a Python exception on error.
inline bool extract_str_list(PyObject* list, Py_ssize_t n,
std::vector<const char*>& out) {
for (Py_ssize_t i = 0; i < n; i++) {
out[i] = PyUnicode_AsUTF8AndSize(PyList_GetItem(list, i), nullptr);
if (out[i] == nullptr) {
return false;
}
}
return true;
}
// Helper: extract a Py_buffer per element of a list[bytes-like] of length n.
// On success, `out` holds n acquired buffers (caller must PyBuffer_Release
// each). On failure, any buffers already acquired are released before
// returning false, and a Python exception is set.
inline bool extract_buffer_list(PyObject* list, Py_ssize_t n, int flags,
std::vector<Py_buffer>& out) {
for (Py_ssize_t i = 0; i < n; i++) {
if (PyObject_GetBuffer(PyList_GetItem(list, i), &out[i], flags) != 0) {
for (Py_ssize_t j = 0; j < i; j++) {
PyBuffer_Release(&out[j]);
}
return false;
}
}
return true;
}
inline void release_buffer_list(std::vector<Py_buffer>& buffers) {
for (auto& buf : buffers) {
PyBuffer_Release(&buf);
}
}
} // namespace
/// @brief Check file existence for a batch of paths.
/// @param paths list[str] – absolute paths to check.
/// @return list[bool] – True if the corresponding path exists, False otherwise.
@@ -51,11 +187,157 @@ static PyObject* batch_lookup(PyObject* /*self*/, PyObject* args) {
return result;
}
/// @brief Store a batch of blocks, each from its own buffer, to disk.
/// @param tmp_paths list[str] – one temp path per block.
/// @param dest_paths list[str] – one destination path per block.
/// @param buffers list[bytes-like] – one source buffer per block.
/// @param use_o_direct bool – whether to open files with O_DIRECT
/// (default True). Ignored where O_DIRECT is unsupported
/// by the platform.
/// @note Releases the GIL for the entire batch. Raises on first error.
static PyObject* batch_store_block(PyObject* /*self*/, PyObject* args) {
PyObject* tmp_paths_obj = nullptr;
PyObject* dest_paths_obj = nullptr;
PyObject* buffers_obj = nullptr;
int use_o_direct = 1;
if (!PyArg_ParseTuple(args, "O!O!O!|p", &PyList_Type, &tmp_paths_obj,
&PyList_Type, &dest_paths_obj, &PyList_Type,
&buffers_obj, &use_o_direct)) {
return nullptr;
}
const Py_ssize_t n = PyList_Size(tmp_paths_obj);
if (PyList_Size(dest_paths_obj) != n || PyList_Size(buffers_obj) != n) {
PyErr_SetString(
PyExc_ValueError,
"tmp_paths, dest_paths and buffers must have the same length");
return nullptr;
}
std::vector<const char*> tmp_paths(n);
std::vector<const char*> dest_paths(n);
if (!extract_str_list(tmp_paths_obj, n, tmp_paths)) return nullptr;
if (!extract_str_list(dest_paths_obj, n, dest_paths)) return nullptr;
std::vector<Py_buffer> buffers(n);
if (!extract_buffer_list(buffers_obj, n, PyBUF_SIMPLE, buffers)) {
return nullptr;
}
Py_ssize_t failed_index = -1;
int failure_errno = 0;
{
Py_BEGIN_ALLOW_THREADS for (Py_ssize_t i = 0; i < n; i++) {
const char* buf = static_cast<const char*>(buffers[i].buf);
const int err =
_store_block(tmp_paths[i], dest_paths[i], buf,
static_cast<size_t>(buffers[i].len), use_o_direct);
if (err != 0) {
failed_index = i;
failure_errno = err;
break;
}
}
Py_END_ALLOW_THREADS
}
release_buffer_list(buffers);
if (failed_index >= 0) {
// PyErr_SetFromErrnoWithFilename() reads the errno to format exception.
errno = failure_errno;
return PyErr_SetFromErrnoWithFilename(PyExc_OSError,
dest_paths[failed_index]);
}
Py_RETURN_NONE;
}
/// @brief Load a batch of blocks from disk, each into its own buffer.
/// @param source_paths list[str] – one source path per block.
/// @param buffers list[writable bytes-like] – one destination buffer
/// per block.
/// @param use_o_direct bool – whether to open files with O_DIRECT
/// (default True). Ignored where O_DIRECT is unsupported
/// by the platform.
/// @note Releases the GIL for the entire batch. Raises on first error.
static PyObject* batch_load_block(PyObject* /*self*/, PyObject* args) {
PyObject* source_paths_obj = nullptr;
PyObject* buffers_obj = nullptr;
int use_o_direct = 1;
if (!PyArg_ParseTuple(args, "O!O!|p", &PyList_Type, &source_paths_obj,
&PyList_Type, &buffers_obj, &use_o_direct)) {
return nullptr;
}
const Py_ssize_t n = PyList_Size(source_paths_obj);
if (PyList_Size(buffers_obj) != n) {
PyErr_SetString(PyExc_ValueError,
"source_paths and buffers must have the same length");
return nullptr;
}
std::vector<const char*> source_paths(n);
if (!extract_str_list(source_paths_obj, n, source_paths)) return nullptr;
std::vector<Py_buffer> buffers(n);
if (!extract_buffer_list(buffers_obj, n, PyBUF_WRITABLE, buffers)) {
return nullptr;
}
Py_ssize_t failed_index = -1;
int failure_errno = 0;
{
Py_BEGIN_ALLOW_THREADS for (Py_ssize_t i = 0; i < n; i++) {
char* buf = static_cast<char*>(buffers[i].buf);
const int err =
_load_block(source_paths[i], buf, static_cast<size_t>(buffers[i].len),
use_o_direct);
if (err != 0) {
failed_index = i;
failure_errno = err;
break;
}
}
Py_END_ALLOW_THREADS
}
release_buffer_list(buffers);
if (failed_index >= 0) {
// PyErr_SetFromErrnoWithFilename() reads the errno to format exception.
errno = failure_errno;
return PyErr_SetFromErrnoWithFilename(PyExc_OSError,
source_paths[failed_index]);
}
Py_RETURN_NONE;
}
static PyMethodDef fs_io_C_methods[] = {
{"batch_lookup", batch_lookup, METH_VARARGS,
"batch_lookup(paths: list[str]) -> list[bool]\n"
"\n"
"Check file existence for a batch of paths."},
{"batch_store_block", batch_store_block, METH_VARARGS,
"batch_store_block(tmp_paths: list[str], dest_paths: list[str],\n"
" buffers: list[bytes-like],\n"
" use_o_direct: bool = True) -> None\n"
"\n"
"Store a batch of blocks, each from its own buffer, to disk. Raises on "
"first error."},
{"batch_load_block", batch_load_block, METH_VARARGS,
"batch_load_block(source_paths: list[str],\n"
" buffers: list[writable bytes-like],\n"
" use_o_direct: bool = True) -> None\n"
"\n"
"Load a batch of blocks from disk into corresponding buffers. "
"Raises on first error."},
{nullptr, nullptr, 0, nullptr},
};
+67 -32
View File
@@ -46,6 +46,7 @@ from vllm.v1.kv_offload.tiering.fs.thread_pool import DualQueueThreadPool
# Helpers
# ---------------------------------------------------------------------------
_NUM_BLOCKS = 8
_BLOCK_ELEMENTS = 128 * mmap.PAGESIZE # 2MB per block for pagesize 4096.
_DTYPE: torch.dtype = torch.float32
_CTX = ReqContext(req_id="test")
@@ -162,7 +163,7 @@ def _page_aligned_rand_tensor(
@pytest.fixture
def fs_tier(tmp_path):
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
tensor = _page_aligned_zero_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS)
mock_view = memoryview(tensor.numpy())
tier = FileSystemTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
@@ -178,7 +179,7 @@ def fs_tier(tmp_path):
@pytest.fixture
def fs_tier_with_events(tmp_path):
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
tensor = _page_aligned_zero_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS)
mock_view = memoryview(tensor.numpy())
tier = FileSystemTierManager(
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
@@ -347,33 +348,43 @@ def test_shutdown_discards_pending_tasks(fs_tier):
assert all(not t.is_alive() for t in tier._pool._threads)
def test_store_load_data_integrity(fs_tier):
"""Data written by store must be exactly recovered by load."""
@pytest.mark.parametrize("batch_size", [0, 1, 2, 5])
@pytest.mark.parametrize("use_c_ext", [True, False])
def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
"""Data written by store must be exactly recovered by load, for batches
of any size -- including the empty batch."""
import vllm.v1.kv_offload.tiering.fs.io as io_mod
if use_c_ext and not io_mod._HAS_FSIO_C:
pytest.skip("fs_io_C extension not built")
monkeypatch.setattr(io_mod, "_HAS_FSIO_C", use_c_ext)
tier, tensor = fs_tier
# Populate tensor with random data
tensor[:] = _page_aligned_rand_tensor(4, _BLOCK_ELEMENTS)
tensor[:] = _page_aligned_rand_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS)
# Store first 2 blocks
num_store = 2
expected = tensor[:num_store].clone()
keys = [key(i) for i in range(batch_size)]
store_block_ids = list(range(batch_size))
load_block_ids = list(range(_NUM_BLOCKS - batch_size, _NUM_BLOCKS))
expected = tensor[:batch_size].clone()
store_ids = list(range(num_store))
keys = [key(i) for i in range(num_store)]
tier.submit_store(make_job(1, keys, store_block_ids))
store_results = drain(tier)
assert len(store_results) == 1
assert store_results[0].success
assert all(os.path.exists(tier.file_mapper.get_file_name(k)) for k in keys)
tier.submit_store(make_job(1, keys, store_ids))
results = drain(tier)
assert all(r.success for r in results)
# reset tensor to prove data is read from disk
tensor[:] = 0.0
# Overwrite source blocks to prove data is read from disk
tensor[:num_store] = 0.0
# Load into a range disjoint by index from the store ids, to also
# exercise loading a block into a different id than it was stored from.
tier.submit_load(make_job(2, keys, load_block_ids, is_promotion=True))
load_results = drain(tier)
assert len(load_results) == 1
assert load_results[0].success
# Load into last 2 blocks
load_ids = [2, 3]
tier.submit_load(make_job(2, keys, load_ids, is_promotion=True))
results = drain(tier)
assert all(r.success for r in results)
for i, bid in enumerate(load_ids):
for i, bid in enumerate(load_block_ids):
assert torch.allclose(tensor[bid], expected[i]), (
f"Block {bid} data mismatch after store+load"
)
@@ -499,6 +510,30 @@ def test_batch_lookup_dispatch(fs_tier, monkeypatch, use_c_ext):
assert results == [LookupResult.HIT, LookupResult.MISS]
@pytest.mark.parametrize("use_c_ext", [True, False])
def test_out_of_bounds_block_id_smoke(fs_tier, monkeypatch, use_c_ext):
"""Smoke test: a block id beyond the primary tensor's block count must
fail the job, for both the C extension and the Python fallback."""
import vllm.v1.kv_offload.tiering.fs.io as io_mod
if use_c_ext and not io_mod._HAS_FSIO_C:
pytest.skip("fs_io_C extension not built")
monkeypatch.setattr(io_mod, "_HAS_FSIO_C", use_c_ext)
tier, tensor = fs_tier
out_of_bounds_bid = tensor.shape[0] # one past the last valid block
tier.submit_store(make_job(1, [key(1)], [out_of_bounds_bid]))
store_results = drain(tier)
assert len(store_results) == 1
assert not store_results[0].success
tier.submit_load(make_job(2, [key(1)], [out_of_bounds_bid], is_promotion=True))
load_results = drain(tier)
assert len(load_results) == 1
assert not load_results[0].success
# ---------------------------------------------------------------------------
# KV events
# ---------------------------------------------------------------------------
@@ -571,14 +606,14 @@ def test_mixed_job_results_emit_event_only_for_successful_job(
tier = fs_tier_with_events
failing_path = tier.file_mapper.get_file_name(key(1))
original_store_block = mgr_mod.store_block
original_batch_store_block = mgr_mod.batch_store_block
def flaky_store_block(dest_path, *args, **kwargs):
if dest_path == failing_path:
def flaky_batch_store_block(paths, *args, **kwargs):
if failing_path in paths:
raise OSError("injected store failure")
return original_store_block(dest_path, *args, **kwargs)
return original_batch_store_block(paths, *args, **kwargs)
monkeypatch.setattr(mgr_mod, "store_block", flaky_store_block)
monkeypatch.setattr(mgr_mod, "batch_store_block", flaky_batch_store_block)
tier.submit_store(make_job(1, [key(1)], [0]))
tier.submit_store(make_job(2, [key(2)], [1]))
@@ -599,14 +634,14 @@ def test_partially_failed_store_emits_no_event(fs_tier_with_events, monkeypatch)
tier = fs_tier_with_events
failing_path = tier.file_mapper.get_file_name(key(2))
original_store_block = mgr_mod.store_block
original_batch_store_block = mgr_mod.batch_store_block
def flaky_store_block(dest_path, *args, **kwargs):
if dest_path == failing_path:
def flaky_batch_store_block(paths, *args, **kwargs):
if failing_path in paths:
raise OSError("injected store failure")
return original_store_block(dest_path, *args, **kwargs)
return original_batch_store_block(paths, *args, **kwargs)
monkeypatch.setattr(mgr_mod, "store_block", flaky_store_block)
monkeypatch.setattr(mgr_mod, "batch_store_block", flaky_batch_store_block)
tier.submit_store(make_job(1, [key(1), key(2)], [0, 1]))
results = drain(tier)
+80 -2
View File
@@ -8,6 +8,18 @@ import os
import random
import threading
try:
from vllm.fs_io_C import ( # pyright: ignore[reportMissingImports]
batch_load_block as batch_load_block_C,
)
from vllm.fs_io_C import (
batch_store_block as batch_store_block_C,
)
_HAS_FSIO_C = True
except ImportError:
_HAS_FSIO_C = False
logger = logging.getLogger(__name__)
# O_DIRECT is Linux-specific and not available on macOS
@@ -59,7 +71,23 @@ def _ensure_dirs(path: str) -> None:
os.makedirs(os.path.dirname(path), exist_ok=True)
def store_block(
def _validate_offsets(view: memoryview, offsets: list[int], block_size: int) -> None:
"""Raise if any block would read/write past the bounds of `view`.
Without this, an out-of-range offset silently clips to a shorter (or
empty) slice instead of failing, since memoryview slicing follows
Python's slice-clamping semantics rather than raising.
"""
total_len = len(view.cast("B"))
for offset in offsets:
if offset < 0 or offset + block_size > total_len:
raise ValueError(
f"block offset {offset} (block_size {block_size}) is out of "
f"bounds for a buffer of size {total_len}"
)
def _store_block(
dest_path: str,
buffer: memoryview,
offset: int,
@@ -104,7 +132,7 @@ def store_block(
raise
def load_block(
def _load_block(
source_path: str,
view: memoryview,
offset: int,
@@ -117,6 +145,7 @@ def load_block(
fd: int | None = None
view_slice = view.cast("B")[offset : offset + block_size]
o_direct = O_DIRECT if use_o_direct else 0
try:
fd = os.open(source_path, os.O_RDONLY | o_direct)
bytes_read = os.readv(fd, [view_slice])
@@ -133,3 +162,52 @@ def load_block(
finally:
if fd is not None:
os.close(fd)
def batch_store_block(
paths: list[str],
view: memoryview,
offsets: list[int],
block_size: int,
use_o_direct: bool = True,
) -> None:
"""
Store a batch of KV blocks from a shared buffer to disk in one call.
Each block buffer[offsets[i] : offsets[i]+block_size] is written atomically
to dest_paths[i] via a temp-file rename. Raises on first error.
"""
_validate_offsets(view, offsets, block_size)
if _HAS_FSIO_C:
view_B = view.cast("B")
view_slices = [view_B[x : x + block_size] for x in offsets]
tmp_paths = [p + _get_tmp_suffix() for p in paths]
return batch_store_block_C(tmp_paths, paths, view_slices, use_o_direct)
else:
for path, offset in zip(paths, offsets):
_store_block(path, view, offset, block_size, use_o_direct)
def batch_load_block(
paths: list[str],
view: memoryview,
offsets: list[int],
block_size: int,
use_o_direct: bool = True,
) -> None:
"""
Load a batch of KV blocks from disk into a shared buffer in one call.
Block i is read from source_paths[i] into view[offsets[i] : offsets[i]+block_size].
Raises on first error and removes the offending file.
"""
_validate_offsets(view, offsets, block_size)
if _HAS_FSIO_C:
view_B = view.cast("B")
view_slices = [view_B[x : x + block_size] for x in offsets]
return batch_load_block_C(paths, view_slices, use_o_direct)
else:
for path, offset in zip(paths, offsets):
_load_block(path, view, offset, block_size, use_o_direct)
+22 -23
View File
@@ -49,7 +49,11 @@ from vllm.v1.kv_offload.tiering.base import (
ScheduleEndContext,
SecondaryTierManager,
)
from vllm.v1.kv_offload.tiering.fs.io import load_block, probe_o_direct, store_block
from vllm.v1.kv_offload.tiering.fs.io import (
batch_load_block,
batch_store_block,
probe_o_direct,
)
from vllm.v1.kv_offload.tiering.fs.thread_pool import DualQueueThreadPool
if TYPE_CHECKING:
@@ -203,33 +207,28 @@ class FileSystemTierManager(SecondaryTierManager):
def submit_store(self, job_metadata: JobMetadata) -> None:
if self.events is not None:
self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys)
tasks = (
functools.partial(
store_block,
self.file_mapper.get_file_name(key),
self._primary_kv_view,
int(bid) * self._block_size,
self._block_size,
self._use_o_direct,
)
for key, bid in zip(job_metadata.keys, job_metadata.block_ids)
task = functools.partial(
batch_store_block,
[self.file_mapper.get_file_name(key) for key in job_metadata.keys],
self._primary_kv_view,
[int(bid) * self._block_size for bid in job_metadata.block_ids],
self._block_size,
self._use_o_direct,
)
self._pool.enqueue_store(job_metadata.job_id, len(job_metadata.keys), tasks)
self._pool.enqueue_store(job_metadata.job_id, 1, [task])
@override
def submit_load(self, job_metadata: JobMetadata) -> None:
tasks = (
functools.partial(
load_block,
self.file_mapper.get_file_name(key),
self._primary_kv_view,
int(bid) * self._block_size,
self._block_size,
self._use_o_direct,
)
for key, bid in zip(job_metadata.keys, job_metadata.block_ids)
task = functools.partial(
batch_load_block,
[self.file_mapper.get_file_name(key) for key in job_metadata.keys],
self._primary_kv_view,
[int(bid) * self._block_size for bid in job_metadata.block_ids],
self._block_size,
self._use_o_direct,
)
self._pool.enqueue_load(job_metadata.job_id, len(job_metadata.keys), tasks)
self._pool.enqueue_load(job_metadata.job_id, 1, [task])
@override
def get_finished_jobs(self) -> Iterable[JobResult]: