KV-Cache multi-tier offloading async batched lookup (#44193)
Signed-off-by: Effi Ofer <effi.ofer@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -0,0 +1,158 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for AsyncLookupManager."""
|
||||
|
||||
import threading
|
||||
from collections.abc import Iterable
|
||||
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, make_offload_key
|
||||
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
||||
|
||||
|
||||
def _key(i: int) -> OffloadKey:
|
||||
return make_offload_key(str(i).encode(), 0)
|
||||
|
||||
|
||||
def _ctx(req_id: str = "r1") -> ReqContext:
|
||||
return ReqContext(req_id=req_id)
|
||||
|
||||
|
||||
class InMemoryLookupManager(AsyncLookupManager):
|
||||
"""Test subclass backed by an in-memory set."""
|
||||
|
||||
def __init__(self, existing_keys: set[OffloadKey] | None = None):
|
||||
super().__init__(tier_type="test")
|
||||
self._existing = existing_keys or set()
|
||||
self._results_ready = threading.Event()
|
||||
|
||||
def batch_lookup(
|
||||
self, keys: list[OffloadKey], req_context: ReqContext
|
||||
) -> Iterable[bool]:
|
||||
results = [k in self._existing for k in keys]
|
||||
self._results_ready.set()
|
||||
return results
|
||||
|
||||
|
||||
class TestAsyncLookupManager:
|
||||
def test_new_key_returns_none(self):
|
||||
mgr = InMemoryLookupManager()
|
||||
assert mgr.lookup(_key(1), _ctx()) is None
|
||||
mgr.shutdown()
|
||||
|
||||
def test_found_key_returns_true(self):
|
||||
mgr = InMemoryLookupManager(existing_keys={_key(1)})
|
||||
assert mgr.lookup(_key(1), _ctx()) is None
|
||||
mgr.flush()
|
||||
mgr._results_ready.wait()
|
||||
mgr._results_ready.clear()
|
||||
assert mgr.lookup(_key(1), _ctx()) is True
|
||||
mgr.shutdown()
|
||||
|
||||
def test_not_found_key_returns_false(self):
|
||||
mgr = InMemoryLookupManager(existing_keys=set())
|
||||
assert mgr.lookup(_key(1), _ctx()) is None
|
||||
mgr.flush()
|
||||
mgr._results_ready.wait()
|
||||
mgr._results_ready.clear()
|
||||
assert mgr.lookup(_key(1), _ctx()) is False
|
||||
mgr.shutdown()
|
||||
|
||||
def test_multiple_keys_single_step(self):
|
||||
existing = {_key(1), _key(3)}
|
||||
mgr = InMemoryLookupManager(existing_keys=existing)
|
||||
ctx = _ctx()
|
||||
for i in range(1, 5):
|
||||
assert mgr.lookup(_key(i), ctx) is None
|
||||
mgr.flush()
|
||||
mgr._results_ready.wait()
|
||||
mgr._results_ready.clear()
|
||||
assert mgr.lookup(_key(1), ctx) is True
|
||||
assert mgr.lookup(_key(2), ctx) is False
|
||||
assert mgr.lookup(_key(3), ctx) is True
|
||||
assert mgr.lookup(_key(4), ctx) is False
|
||||
mgr.shutdown()
|
||||
|
||||
def test_cleanup_removes_entries(self):
|
||||
mgr = InMemoryLookupManager(existing_keys={_key(1)})
|
||||
ctx = _ctx("req_a")
|
||||
mgr.lookup(_key(1), ctx)
|
||||
mgr.flush()
|
||||
mgr._results_ready.wait()
|
||||
mgr._results_ready.clear()
|
||||
assert mgr.lookup(_key(1), ctx) is True
|
||||
mgr.cleanup("req_a")
|
||||
assert _key(1) not in mgr._lookup_state
|
||||
mgr.shutdown()
|
||||
|
||||
def test_cleanup_preserves_shared_entries(self):
|
||||
mgr = InMemoryLookupManager(existing_keys={_key(1)})
|
||||
ctx_a = _ctx("req_a")
|
||||
ctx_b = _ctx("req_b")
|
||||
mgr.lookup(_key(1), ctx_a)
|
||||
mgr.lookup(_key(1), ctx_b)
|
||||
mgr.flush()
|
||||
mgr._results_ready.wait()
|
||||
mgr._results_ready.clear()
|
||||
# Drain so result is applied
|
||||
mgr.lookup(_key(1), ctx_a)
|
||||
mgr.cleanup("req_a")
|
||||
# Key still present because req_b still references it
|
||||
assert _key(1) in mgr._lookup_state
|
||||
mgr.cleanup("req_b")
|
||||
assert _key(1) not in mgr._lookup_state
|
||||
mgr.shutdown()
|
||||
|
||||
def test_flush_no_queue_post_when_empty(self):
|
||||
mgr = InMemoryLookupManager()
|
||||
mgr.flush()
|
||||
assert mgr._lookup_queue.empty()
|
||||
mgr.shutdown()
|
||||
|
||||
def test_repeated_lookup_same_key_no_duplicate_batch(self):
|
||||
mgr = InMemoryLookupManager(existing_keys={_key(1)})
|
||||
ctx = _ctx()
|
||||
mgr.lookup(_key(1), ctx)
|
||||
mgr.lookup(_key(1), ctx)
|
||||
assert len(mgr._lookup_batch) == 1
|
||||
mgr.shutdown()
|
||||
|
||||
def test_cleanup_unknown_req_id_is_noop(self):
|
||||
mgr = InMemoryLookupManager(existing_keys={_key(1)})
|
||||
ctx = _ctx("req_a")
|
||||
mgr.lookup(_key(1), ctx)
|
||||
mgr.flush()
|
||||
mgr._results_ready.wait()
|
||||
mgr._results_ready.clear()
|
||||
mgr.lookup(_key(1), ctx)
|
||||
mgr.cleanup("nonexistent")
|
||||
assert _key(1) in mgr._lookup_state
|
||||
mgr.shutdown()
|
||||
|
||||
def test_multiple_flushes_across_steps(self):
|
||||
existing = {_key(1), _key(2), _key(3)}
|
||||
mgr = InMemoryLookupManager(existing_keys=existing)
|
||||
ctx = _ctx()
|
||||
|
||||
# Step 1: lookup key 1, flush
|
||||
mgr.lookup(_key(1), ctx)
|
||||
mgr.flush()
|
||||
mgr._results_ready.wait()
|
||||
mgr._results_ready.clear()
|
||||
|
||||
# Step 2: lookup keys 2 and 3, flush
|
||||
mgr.lookup(_key(2), ctx)
|
||||
mgr.lookup(_key(3), ctx)
|
||||
mgr.flush()
|
||||
mgr._results_ready.wait()
|
||||
mgr._results_ready.clear()
|
||||
|
||||
# All results should be available
|
||||
assert mgr.lookup(_key(1), ctx) is True
|
||||
assert mgr.lookup(_key(2), ctx) is True
|
||||
assert mgr.lookup(_key(3), ctx) is True
|
||||
mgr.shutdown()
|
||||
|
||||
def test_shutdown_unblocks_worker(self):
|
||||
mgr = InMemoryLookupManager()
|
||||
mgr.shutdown()
|
||||
assert not mgr._thread.is_alive()
|
||||
@@ -71,9 +71,9 @@ def make_job(
|
||||
)
|
||||
|
||||
|
||||
def drain(tier: FileSystemTierManager, max_rounds: int = 40) -> list:
|
||||
def drain(tier: FileSystemTierManager, max_rounds: int = 100) -> list:
|
||||
"""
|
||||
Call get_finished_jobs() repeatedly until no new results arrive for 5
|
||||
Call get_finished_jobs() repeatedly until no new results arrive for 20
|
||||
consecutive rounds or max_rounds is reached.
|
||||
"""
|
||||
results = []
|
||||
@@ -86,11 +86,29 @@ def drain(tier: FileSystemTierManager, max_rounds: int = 40) -> list:
|
||||
idle = 0
|
||||
else:
|
||||
idle += 1
|
||||
if idle >= 5:
|
||||
if idle >= 20:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
def lookup_and_wait(
|
||||
tier: FileSystemTierManager,
|
||||
keys: list[OffloadKey],
|
||||
ctx: ReqContext = _CTX,
|
||||
timeout: float = 1.0,
|
||||
) -> list[bool]:
|
||||
"""Perform a full async lookup cycle and return resolved results."""
|
||||
for k in keys:
|
||||
tier.lookup(k, ctx)
|
||||
tier.on_schedule_end()
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if not tier._lookup_manager._pending_results.empty():
|
||||
break
|
||||
time.sleep(0.01)
|
||||
return [tier.lookup(k, ctx) for k in keys]
|
||||
|
||||
|
||||
def _page_aligned_zero_tensor(
|
||||
num_blocks: int, block_elements: int, dtype: torch.dtype = _DTYPE
|
||||
) -> torch.Tensor:
|
||||
@@ -145,8 +163,8 @@ def fs_tier(tmp_path):
|
||||
|
||||
def test_lookup_empty_tier(fs_tier):
|
||||
tier, _ = fs_tier
|
||||
assert tier.lookup(key(1), _CTX) is False
|
||||
assert tier.lookup(key(2), _CTX) is False
|
||||
results = lookup_and_wait(tier, [key(1), key(2)])
|
||||
assert results == [False, False]
|
||||
|
||||
|
||||
def test_store_creates_file_and_lookup_succeeds(fs_tier):
|
||||
@@ -156,7 +174,7 @@ def test_store_creates_file_and_lookup_succeeds(fs_tier):
|
||||
results = drain(tier)
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
assert tier.lookup(key(1), _CTX) is True
|
||||
assert lookup_and_wait(tier, [key(1)]) == [True]
|
||||
dest = tier.file_mapper.get_file_name(key(1))
|
||||
assert os.path.exists(dest), f"Expected file at {dest}"
|
||||
|
||||
@@ -168,16 +186,14 @@ def test_store_then_load_roundtrip(fs_tier):
|
||||
store_results = drain(tier)
|
||||
assert all(r.success for r in store_results)
|
||||
|
||||
assert tier.lookup(key(1), _CTX) is True
|
||||
assert tier.lookup(key(2), _CTX) is True
|
||||
assert lookup_and_wait(tier, [key(1), key(2)]) == [True, True]
|
||||
|
||||
job_l = make_job(2, [key(1), key(2)], [2, 3], is_promotion=True)
|
||||
tier.submit_load(job_l)
|
||||
load_results = drain(tier)
|
||||
assert all(r.success for r in load_results)
|
||||
# Blocks stay on disk after load
|
||||
assert tier.lookup(key(1), _CTX) is True
|
||||
assert tier.lookup(key(2), _CTX) is True
|
||||
assert lookup_and_wait(tier, [key(1), key(2)]) == [True, True]
|
||||
|
||||
|
||||
def test_invalid_path_raises_at_construction():
|
||||
@@ -213,8 +229,7 @@ def test_multiple_jobs_tracked_independently(fs_tier):
|
||||
results = drain(tier)
|
||||
job_ids = {r.job_id for r in results}
|
||||
assert job_ids == {1, 2}
|
||||
assert tier.lookup(key(1), _CTX) is True
|
||||
assert tier.lookup(key(2), _CTX) is True
|
||||
assert lookup_and_wait(tier, [key(1), key(2)]) == [True, True]
|
||||
|
||||
|
||||
def test_multi_block_job_partial_failure(fs_tier):
|
||||
|
||||
@@ -8,6 +8,7 @@ without S3 credentials or a live object store. They verify the manager's
|
||||
state machine: job submission, transfer completion polling, and lookup.
|
||||
"""
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from types import SimpleNamespace
|
||||
@@ -206,6 +207,24 @@ def drain(
|
||||
return results
|
||||
|
||||
|
||||
def lookup_and_wait(
|
||||
tier: ObjectStoreSecondaryTierManager,
|
||||
keys: list[OffloadKey],
|
||||
ctx: ReqContext = _CTX,
|
||||
timeout: float = 1.0,
|
||||
) -> list[bool]:
|
||||
"""Perform a full async lookup cycle and return resolved results."""
|
||||
for k in keys:
|
||||
tier.lookup(k, ctx)
|
||||
tier.on_schedule_end()
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if not tier._lookup_manager._pending_results.empty():
|
||||
break
|
||||
time.sleep(0.01)
|
||||
return [tier.lookup(k, ctx) for k in keys]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -216,19 +235,19 @@ class TestMockObjTierBasic:
|
||||
self.tier, self.agent = _make_tier(num_blocks=4)
|
||||
|
||||
def test_lookup_empty_tier(self):
|
||||
assert self.tier.lookup(key(1), _CTX) is False
|
||||
assert lookup_and_wait(self.tier, [key(1)]) == [False]
|
||||
|
||||
def test_store_and_lookup(self):
|
||||
self.tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
results = drain(self.tier)
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
assert self.tier.lookup(key(1), _CTX) is True
|
||||
assert lookup_and_wait(self.tier, [key(1)]) == [True]
|
||||
|
||||
def test_lookup_unrelated_key_returns_false(self):
|
||||
self.tier.submit_store(make_job(1, [key(1)], [0]))
|
||||
drain(self.tier)
|
||||
assert self.tier.lookup(key(999), _CTX) is False
|
||||
assert lookup_and_wait(self.tier, [key(999)]) == [False]
|
||||
|
||||
def test_store_then_load_roundtrip(self):
|
||||
self.tier.submit_store(make_job(1, [key(1), key(2)], [0, 1]))
|
||||
@@ -280,15 +299,13 @@ class TestMockObjTierMultiBlock:
|
||||
results = drain(tier)
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
assert all(tier.lookup(k, _CTX) for k in keys)
|
||||
assert lookup_and_wait(tier, keys) == [True] * 8
|
||||
|
||||
def test_partial_block_lookup(self):
|
||||
tier, _ = _make_tier(num_blocks=4)
|
||||
tier.submit_store(make_job(1, [key(0), key(1)], [0, 1]))
|
||||
drain(tier)
|
||||
assert tier.lookup(key(0), _CTX) is True
|
||||
assert tier.lookup(key(1), _CTX) is True
|
||||
assert tier.lookup(key(2), _CTX) is False
|
||||
assert lookup_and_wait(tier, [key(0), key(1), key(2)]) == [True, True, False]
|
||||
|
||||
|
||||
class TestMockObjTierFailures:
|
||||
@@ -297,7 +314,7 @@ class TestMockObjTierFailures:
|
||||
agent.query_memory = lambda *a, **k: (_ for _ in ()).throw(
|
||||
RuntimeError("backend error")
|
||||
)
|
||||
assert tier.lookup(key(1), _CTX) is False
|
||||
assert lookup_and_wait(tier, [key(1)]) == [False]
|
||||
|
||||
def test_submit_store_register_memory_failure_reported_in_get_finished(self):
|
||||
tier, agent = _make_tier(num_blocks=4)
|
||||
|
||||
Reference in New Issue
Block a user