[KV-offload][FS] : Batch store/load_block in C (#49152)

Signed-off-by: <>
Co-authored-by: Varun Sundar Rabindranath <varun-sundar-rabindranath@h100-01.nemg-001.lab.rdu2.dc.redhat.com>
This commit is contained in:
Varun Sundar Rabindranath
2026-07-28 11:21:53 +08:00
committed by GitHub
co-authored by Varun Sundar Rabindranath
parent 60915c972c
commit d18ed2304a
4 changed files with 452 additions and 58 deletions
+67 -32
View File
@@ -46,6 +46,7 @@ from vllm.v1.kv_offload.tiering.fs.thread_pool import DualQueueThreadPool
# Helpers
# ---------------------------------------------------------------------------
_NUM_BLOCKS = 8
_BLOCK_ELEMENTS = 128 * mmap.PAGESIZE # 2MB per block for pagesize 4096.
_DTYPE: torch.dtype = torch.float32
_CTX = ReqContext(req_id="test")
@@ -162,7 +163,7 @@ def _page_aligned_rand_tensor(
@pytest.fixture
def fs_tier(tmp_path):
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
tensor = _page_aligned_zero_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS)
mock_view = memoryview(tensor.numpy())
tier = FileSystemTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
@@ -178,7 +179,7 @@ def fs_tier(tmp_path):
@pytest.fixture
def fs_tier_with_events(tmp_path):
tensor = _page_aligned_zero_tensor(4, _BLOCK_ELEMENTS)
tensor = _page_aligned_zero_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS)
mock_view = memoryview(tensor.numpy())
tier = FileSystemTierManager(
offloading_spec=_make_offloading_spec(enable_kv_cache_events=True),
@@ -347,33 +348,43 @@ def test_shutdown_discards_pending_tasks(fs_tier):
assert all(not t.is_alive() for t in tier._pool._threads)
def test_store_load_data_integrity(fs_tier):
"""Data written by store must be exactly recovered by load."""
@pytest.mark.parametrize("batch_size", [0, 1, 2, 5])
@pytest.mark.parametrize("use_c_ext", [True, False])
def test_store_load_data_integrity(fs_tier, monkeypatch, use_c_ext, batch_size):
"""Data written by store must be exactly recovered by load, for batches
of any size -- including the empty batch."""
import vllm.v1.kv_offload.tiering.fs.io as io_mod
if use_c_ext and not io_mod._HAS_FSIO_C:
pytest.skip("fs_io_C extension not built")
monkeypatch.setattr(io_mod, "_HAS_FSIO_C", use_c_ext)
tier, tensor = fs_tier
# Populate tensor with random data
tensor[:] = _page_aligned_rand_tensor(4, _BLOCK_ELEMENTS)
tensor[:] = _page_aligned_rand_tensor(_NUM_BLOCKS, _BLOCK_ELEMENTS)
# Store first 2 blocks
num_store = 2
expected = tensor[:num_store].clone()
keys = [key(i) for i in range(batch_size)]
store_block_ids = list(range(batch_size))
load_block_ids = list(range(_NUM_BLOCKS - batch_size, _NUM_BLOCKS))
expected = tensor[:batch_size].clone()
store_ids = list(range(num_store))
keys = [key(i) for i in range(num_store)]
tier.submit_store(make_job(1, keys, store_block_ids))
store_results = drain(tier)
assert len(store_results) == 1
assert store_results[0].success
assert all(os.path.exists(tier.file_mapper.get_file_name(k)) for k in keys)
tier.submit_store(make_job(1, keys, store_ids))
results = drain(tier)
assert all(r.success for r in results)
# reset tensor to prove data is read from disk
tensor[:] = 0.0
# Overwrite source blocks to prove data is read from disk
tensor[:num_store] = 0.0
# Load into a range disjoint by index from the store ids, to also
# exercise loading a block into a different id than it was stored from.
tier.submit_load(make_job(2, keys, load_block_ids, is_promotion=True))
load_results = drain(tier)
assert len(load_results) == 1
assert load_results[0].success
# Load into last 2 blocks
load_ids = [2, 3]
tier.submit_load(make_job(2, keys, load_ids, is_promotion=True))
results = drain(tier)
assert all(r.success for r in results)
for i, bid in enumerate(load_ids):
for i, bid in enumerate(load_block_ids):
assert torch.allclose(tensor[bid], expected[i]), (
f"Block {bid} data mismatch after store+load"
)
@@ -499,6 +510,30 @@ def test_batch_lookup_dispatch(fs_tier, monkeypatch, use_c_ext):
assert results == [LookupResult.HIT, LookupResult.MISS]
@pytest.mark.parametrize("use_c_ext", [True, False])
def test_out_of_bounds_block_id_smoke(fs_tier, monkeypatch, use_c_ext):
"""Smoke test: a block id beyond the primary tensor's block count must
fail the job, for both the C extension and the Python fallback."""
import vllm.v1.kv_offload.tiering.fs.io as io_mod
if use_c_ext and not io_mod._HAS_FSIO_C:
pytest.skip("fs_io_C extension not built")
monkeypatch.setattr(io_mod, "_HAS_FSIO_C", use_c_ext)
tier, tensor = fs_tier
out_of_bounds_bid = tensor.shape[0] # one past the last valid block
tier.submit_store(make_job(1, [key(1)], [out_of_bounds_bid]))
store_results = drain(tier)
assert len(store_results) == 1
assert not store_results[0].success
tier.submit_load(make_job(2, [key(1)], [out_of_bounds_bid], is_promotion=True))
load_results = drain(tier)
assert len(load_results) == 1
assert not load_results[0].success
# ---------------------------------------------------------------------------
# KV events
# ---------------------------------------------------------------------------
@@ -571,14 +606,14 @@ def test_mixed_job_results_emit_event_only_for_successful_job(
tier = fs_tier_with_events
failing_path = tier.file_mapper.get_file_name(key(1))
original_store_block = mgr_mod.store_block
original_batch_store_block = mgr_mod.batch_store_block
def flaky_store_block(dest_path, *args, **kwargs):
if dest_path == failing_path:
def flaky_batch_store_block(paths, *args, **kwargs):
if failing_path in paths:
raise OSError("injected store failure")
return original_store_block(dest_path, *args, **kwargs)
return original_batch_store_block(paths, *args, **kwargs)
monkeypatch.setattr(mgr_mod, "store_block", flaky_store_block)
monkeypatch.setattr(mgr_mod, "batch_store_block", flaky_batch_store_block)
tier.submit_store(make_job(1, [key(1)], [0]))
tier.submit_store(make_job(2, [key(2)], [1]))
@@ -599,14 +634,14 @@ def test_partially_failed_store_emits_no_event(fs_tier_with_events, monkeypatch)
tier = fs_tier_with_events
failing_path = tier.file_mapper.get_file_name(key(2))
original_store_block = mgr_mod.store_block
original_batch_store_block = mgr_mod.batch_store_block
def flaky_store_block(dest_path, *args, **kwargs):
if dest_path == failing_path:
def flaky_batch_store_block(paths, *args, **kwargs):
if failing_path in paths:
raise OSError("injected store failure")
return original_store_block(dest_path, *args, **kwargs)
return original_batch_store_block(paths, *args, **kwargs)
monkeypatch.setattr(mgr_mod, "store_block", flaky_store_block)
monkeypatch.setattr(mgr_mod, "batch_store_block", flaky_batch_store_block)
tier.submit_store(make_job(1, [key(1), key(2)], [0, 1]))
results = drain(tier)