KV-Cache multi-tier offloading async batched lookup (#44193)

Signed-off-by: Effi Ofer <effi.ofer@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Effi Ofer
2026-06-10 14:59:30 +00:00
committed by GitHub
co-authored by Or Ozeri
parent 3cc9fecd58
commit af65e08fc5
6 changed files with 518 additions and 31 deletions
@@ -0,0 +1,158 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for AsyncLookupManager."""
import threading
from collections.abc import Iterable
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, make_offload_key
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
def _key(i: int) -> OffloadKey:
return make_offload_key(str(i).encode(), 0)
def _ctx(req_id: str = "r1") -> ReqContext:
return ReqContext(req_id=req_id)
class InMemoryLookupManager(AsyncLookupManager):
"""Test subclass backed by an in-memory set."""
def __init__(self, existing_keys: set[OffloadKey] | None = None):
super().__init__(tier_type="test")
self._existing = existing_keys or set()
self._results_ready = threading.Event()
def batch_lookup(
self, keys: list[OffloadKey], req_context: ReqContext
) -> Iterable[bool]:
results = [k in self._existing for k in keys]
self._results_ready.set()
return results
class TestAsyncLookupManager:
def test_new_key_returns_none(self):
mgr = InMemoryLookupManager()
assert mgr.lookup(_key(1), _ctx()) is None
mgr.shutdown()
def test_found_key_returns_true(self):
mgr = InMemoryLookupManager(existing_keys={_key(1)})
assert mgr.lookup(_key(1), _ctx()) is None
mgr.flush()
mgr._results_ready.wait()
mgr._results_ready.clear()
assert mgr.lookup(_key(1), _ctx()) is True
mgr.shutdown()
def test_not_found_key_returns_false(self):
mgr = InMemoryLookupManager(existing_keys=set())
assert mgr.lookup(_key(1), _ctx()) is None
mgr.flush()
mgr._results_ready.wait()
mgr._results_ready.clear()
assert mgr.lookup(_key(1), _ctx()) is False
mgr.shutdown()
def test_multiple_keys_single_step(self):
existing = {_key(1), _key(3)}
mgr = InMemoryLookupManager(existing_keys=existing)
ctx = _ctx()
for i in range(1, 5):
assert mgr.lookup(_key(i), ctx) is None
mgr.flush()
mgr._results_ready.wait()
mgr._results_ready.clear()
assert mgr.lookup(_key(1), ctx) is True
assert mgr.lookup(_key(2), ctx) is False
assert mgr.lookup(_key(3), ctx) is True
assert mgr.lookup(_key(4), ctx) is False
mgr.shutdown()
def test_cleanup_removes_entries(self):
mgr = InMemoryLookupManager(existing_keys={_key(1)})
ctx = _ctx("req_a")
mgr.lookup(_key(1), ctx)
mgr.flush()
mgr._results_ready.wait()
mgr._results_ready.clear()
assert mgr.lookup(_key(1), ctx) is True
mgr.cleanup("req_a")
assert _key(1) not in mgr._lookup_state
mgr.shutdown()
def test_cleanup_preserves_shared_entries(self):
mgr = InMemoryLookupManager(existing_keys={_key(1)})
ctx_a = _ctx("req_a")
ctx_b = _ctx("req_b")
mgr.lookup(_key(1), ctx_a)
mgr.lookup(_key(1), ctx_b)
mgr.flush()
mgr._results_ready.wait()
mgr._results_ready.clear()
# Drain so result is applied
mgr.lookup(_key(1), ctx_a)
mgr.cleanup("req_a")
# Key still present because req_b still references it
assert _key(1) in mgr._lookup_state
mgr.cleanup("req_b")
assert _key(1) not in mgr._lookup_state
mgr.shutdown()
def test_flush_no_queue_post_when_empty(self):
mgr = InMemoryLookupManager()
mgr.flush()
assert mgr._lookup_queue.empty()
mgr.shutdown()
def test_repeated_lookup_same_key_no_duplicate_batch(self):
mgr = InMemoryLookupManager(existing_keys={_key(1)})
ctx = _ctx()
mgr.lookup(_key(1), ctx)
mgr.lookup(_key(1), ctx)
assert len(mgr._lookup_batch) == 1
mgr.shutdown()
def test_cleanup_unknown_req_id_is_noop(self):
mgr = InMemoryLookupManager(existing_keys={_key(1)})
ctx = _ctx("req_a")
mgr.lookup(_key(1), ctx)
mgr.flush()
mgr._results_ready.wait()
mgr._results_ready.clear()
mgr.lookup(_key(1), ctx)
mgr.cleanup("nonexistent")
assert _key(1) in mgr._lookup_state
mgr.shutdown()
def test_multiple_flushes_across_steps(self):
existing = {_key(1), _key(2), _key(3)}
mgr = InMemoryLookupManager(existing_keys=existing)
ctx = _ctx()
# Step 1: lookup key 1, flush
mgr.lookup(_key(1), ctx)
mgr.flush()
mgr._results_ready.wait()
mgr._results_ready.clear()
# Step 2: lookup keys 2 and 3, flush
mgr.lookup(_key(2), ctx)
mgr.lookup(_key(3), ctx)
mgr.flush()
mgr._results_ready.wait()
mgr._results_ready.clear()
# All results should be available
assert mgr.lookup(_key(1), ctx) is True
assert mgr.lookup(_key(2), ctx) is True
assert mgr.lookup(_key(3), ctx) is True
mgr.shutdown()
def test_shutdown_unblocks_worker(self):
mgr = InMemoryLookupManager()
mgr.shutdown()
assert not mgr._thread.is_alive()
+27 -12
View File
@@ -71,9 +71,9 @@ def make_job(
)
def drain(tier: FileSystemTierManager, max_rounds: int = 40) -> list:
def drain(tier: FileSystemTierManager, max_rounds: int = 100) -> list:
"""
Call get_finished_jobs() repeatedly until no new results arrive for 5
Call get_finished_jobs() repeatedly until no new results arrive for 20
consecutive rounds or max_rounds is reached.
"""
results = []
@@ -86,11 +86,29 @@ def drain(tier: FileSystemTierManager, max_rounds: int = 40) -> list:
idle = 0
else:
idle += 1
if idle >= 5:
if idle >= 20:
break
return results
def lookup_and_wait(
tier: FileSystemTierManager,
keys: list[OffloadKey],
ctx: ReqContext = _CTX,
timeout: float = 1.0,
) -> list[bool]:
"""Perform a full async lookup cycle and return resolved results."""
for k in keys:
tier.lookup(k, ctx)
tier.on_schedule_end()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if not tier._lookup_manager._pending_results.empty():
break
time.sleep(0.01)
return [tier.lookup(k, ctx) for k in keys]
def _page_aligned_zero_tensor(
num_blocks: int, block_elements: int, dtype: torch.dtype = _DTYPE
) -> torch.Tensor:
@@ -145,8 +163,8 @@ def fs_tier(tmp_path):
def test_lookup_empty_tier(fs_tier):
tier, _ = fs_tier
assert tier.lookup(key(1), _CTX) is False
assert tier.lookup(key(2), _CTX) is False
results = lookup_and_wait(tier, [key(1), key(2)])
assert results == [False, False]
def test_store_creates_file_and_lookup_succeeds(fs_tier):
@@ -156,7 +174,7 @@ def test_store_creates_file_and_lookup_succeeds(fs_tier):
results = drain(tier)
assert len(results) == 1
assert results[0].success
assert tier.lookup(key(1), _CTX) is True
assert lookup_and_wait(tier, [key(1)]) == [True]
dest = tier.file_mapper.get_file_name(key(1))
assert os.path.exists(dest), f"Expected file at {dest}"
@@ -168,16 +186,14 @@ def test_store_then_load_roundtrip(fs_tier):
store_results = drain(tier)
assert all(r.success for r in store_results)
assert tier.lookup(key(1), _CTX) is True
assert tier.lookup(key(2), _CTX) is True
assert lookup_and_wait(tier, [key(1), key(2)]) == [True, True]
job_l = make_job(2, [key(1), key(2)], [2, 3], is_promotion=True)
tier.submit_load(job_l)
load_results = drain(tier)
assert all(r.success for r in load_results)
# Blocks stay on disk after load
assert tier.lookup(key(1), _CTX) is True
assert tier.lookup(key(2), _CTX) is True
assert lookup_and_wait(tier, [key(1), key(2)]) == [True, True]
def test_invalid_path_raises_at_construction():
@@ -213,8 +229,7 @@ def test_multiple_jobs_tracked_independently(fs_tier):
results = drain(tier)
job_ids = {r.job_id for r in results}
assert job_ids == {1, 2}
assert tier.lookup(key(1), _CTX) is True
assert tier.lookup(key(2), _CTX) is True
assert lookup_and_wait(tier, [key(1), key(2)]) == [True, True]
def test_multi_block_job_partial_failure(fs_tier):
+25 -8
View File
@@ -8,6 +8,7 @@ without S3 credentials or a live object store. They verify the manager's
state machine: job submission, transfer completion polling, and lookup.
"""
import time
import uuid
from collections.abc import Callable
from types import SimpleNamespace
@@ -206,6 +207,24 @@ def drain(
return results
def lookup_and_wait(
tier: ObjectStoreSecondaryTierManager,
keys: list[OffloadKey],
ctx: ReqContext = _CTX,
timeout: float = 1.0,
) -> list[bool]:
"""Perform a full async lookup cycle and return resolved results."""
for k in keys:
tier.lookup(k, ctx)
tier.on_schedule_end()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if not tier._lookup_manager._pending_results.empty():
break
time.sleep(0.01)
return [tier.lookup(k, ctx) for k in keys]
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
@@ -216,19 +235,19 @@ class TestMockObjTierBasic:
self.tier, self.agent = _make_tier(num_blocks=4)
def test_lookup_empty_tier(self):
assert self.tier.lookup(key(1), _CTX) is False
assert lookup_and_wait(self.tier, [key(1)]) == [False]
def test_store_and_lookup(self):
self.tier.submit_store(make_job(1, [key(1)], [0]))
results = drain(self.tier)
assert len(results) == 1
assert results[0].success
assert self.tier.lookup(key(1), _CTX) is True
assert lookup_and_wait(self.tier, [key(1)]) == [True]
def test_lookup_unrelated_key_returns_false(self):
self.tier.submit_store(make_job(1, [key(1)], [0]))
drain(self.tier)
assert self.tier.lookup(key(999), _CTX) is False
assert lookup_and_wait(self.tier, [key(999)]) == [False]
def test_store_then_load_roundtrip(self):
self.tier.submit_store(make_job(1, [key(1), key(2)], [0, 1]))
@@ -280,15 +299,13 @@ class TestMockObjTierMultiBlock:
results = drain(tier)
assert len(results) == 1
assert results[0].success
assert all(tier.lookup(k, _CTX) for k in keys)
assert lookup_and_wait(tier, keys) == [True] * 8
def test_partial_block_lookup(self):
tier, _ = _make_tier(num_blocks=4)
tier.submit_store(make_job(1, [key(0), key(1)], [0, 1]))
drain(tier)
assert tier.lookup(key(0), _CTX) is True
assert tier.lookup(key(1), _CTX) is True
assert tier.lookup(key(2), _CTX) is False
assert lookup_and_wait(tier, [key(0), key(1), key(2)]) == [True, True, False]
class TestMockObjTierFailures:
@@ -297,7 +314,7 @@ class TestMockObjTierFailures:
agent.query_memory = lambda *a, **k: (_ for _ in ()).throw(
RuntimeError("backend error")
)
assert tier.lookup(key(1), _CTX) is False
assert lookup_and_wait(tier, [key(1)]) == [False]
def test_submit_store_register_memory_failure_reported_in_get_finished(self):
tier, agent = _make_tier(num_blocks=4)