[Bugfix][KV Offload] Reuse in-flight async lookup probes (#55823)
Signed-off-by: Alex <jihui.huang@daocloud.io> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
OpenAI Codex
Or Ozeri
parent
a5f6f61a8a
commit
c612e2bff0
@@ -8,7 +8,7 @@ from collections.abc import Iterable
|
||||
import pytest
|
||||
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, make_offload_key
|
||||
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
||||
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager, LookupPhase
|
||||
|
||||
|
||||
def _key(i: int) -> OffloadKey:
|
||||
@@ -46,10 +46,13 @@ class TestAsyncLookupManager:
|
||||
def test_found_key_returns_true(self):
|
||||
mgr = InMemoryLookupManager(existing_keys={_key(1)})
|
||||
assert mgr.lookup(_key(1), _ctx()) is None
|
||||
assert mgr._lookup_state[_key(1)].phase is LookupPhase.PENDING
|
||||
mgr.flush()
|
||||
assert mgr._lookup_state[_key(1)].phase is LookupPhase.IN_FLIGHT
|
||||
mgr._results_ready.wait()
|
||||
mgr._results_ready.clear()
|
||||
assert mgr.lookup(_key(1), _ctx()) is True
|
||||
assert mgr._lookup_state[_key(1)].phase is LookupPhase.RESOLVED
|
||||
mgr.shutdown()
|
||||
|
||||
def test_not_found_key_returns_false(self):
|
||||
@@ -106,6 +109,68 @@ class TestAsyncLookupManager:
|
||||
assert _key(1) not in mgr._lookup_state
|
||||
mgr.shutdown()
|
||||
|
||||
def test_cleanup_reuses_in_flight_probe(self, monkeypatch: pytest.MonkeyPatch):
|
||||
"""A replacement request shares the in-flight probe and its verdict."""
|
||||
key = _key(1)
|
||||
mgr = InMemoryLookupManager(existing_keys={key})
|
||||
ctx_b = _ctx("req_b")
|
||||
probe_started = threading.Event()
|
||||
release_probe = threading.Event()
|
||||
batch_lookup = mgr.batch_lookup
|
||||
|
||||
def blocking_lookup(keys, req_context):
|
||||
probe_started.set()
|
||||
if not release_probe.wait(timeout=5):
|
||||
raise TimeoutError("Test did not release the backend probe")
|
||||
return batch_lookup(keys, req_context)
|
||||
|
||||
monkeypatch.setattr(mgr, "batch_lookup", blocking_lookup)
|
||||
try:
|
||||
assert mgr.lookup(key, _ctx("req_a")) is None
|
||||
mgr.flush()
|
||||
assert probe_started.wait(timeout=5)
|
||||
assert mgr._lookup_state[key].phase is LookupPhase.IN_FLIGHT
|
||||
mgr.cleanup("req_a")
|
||||
assert mgr.lookup(key, ctx_b) is None
|
||||
assert mgr._lookup_state[key].phase is LookupPhase.IN_FLIGHT
|
||||
mgr.flush()
|
||||
|
||||
release_probe.set()
|
||||
batch = mgr._pending_results.get(timeout=5)
|
||||
mgr._pending_results.put(batch)
|
||||
replacement_result = mgr.lookup(key, ctx_b)
|
||||
finally:
|
||||
release_probe.set()
|
||||
mgr.shutdown()
|
||||
|
||||
assert mgr.batch_lookup_calls == 1
|
||||
assert replacement_result is True
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reclaim_at_shutdown", [False, True], ids=["flush", "shutdown"]
|
||||
)
|
||||
def test_unclaimed_probe_reclaimed_without_lookup(self, reclaim_at_shutdown: bool):
|
||||
"""A completed orphan is released by flush or shutdown without a lookup."""
|
||||
key = _key(1)
|
||||
mgr = InMemoryLookupManager(existing_keys={key})
|
||||
try:
|
||||
mgr.lookup(key, _ctx("req_a"))
|
||||
mgr.flush()
|
||||
mgr.cleanup("req_a")
|
||||
assert key in mgr._lookup_state
|
||||
assert not mgr._req_keys
|
||||
|
||||
if not reclaim_at_shutdown:
|
||||
batch = mgr._pending_results.get(timeout=5)
|
||||
mgr._pending_results.put(batch)
|
||||
mgr.flush()
|
||||
assert key not in mgr._lookup_state
|
||||
finally:
|
||||
mgr.shutdown()
|
||||
|
||||
assert not mgr._lookup_state
|
||||
assert mgr._pending_results.empty()
|
||||
|
||||
def test_stale_result_ignored_after_cleanup_and_key_reuse(self):
|
||||
key = _key(1)
|
||||
mgr = InMemoryLookupManager()
|
||||
@@ -119,11 +184,12 @@ class TestAsyncLookupManager:
|
||||
generation = mgr._lookup_state[key].generation
|
||||
assert generation != stale_generation
|
||||
|
||||
mgr.flush()
|
||||
mgr._results_ready.wait()
|
||||
mgr._results_ready.clear()
|
||||
current_result = mgr._pending_results.get(timeout=5)
|
||||
mgr._pending_results.put([(key, stale_generation, True)])
|
||||
mgr.drain_results()
|
||||
assert mgr.lookup(key, ctx_b) is None
|
||||
|
||||
mgr._pending_results.put([(key, generation, False)])
|
||||
mgr._pending_results.put(current_result)
|
||||
mgr.drain_results()
|
||||
assert mgr.lookup(key, ctx_b) is False
|
||||
mgr.shutdown()
|
||||
|
||||
Reference in New Issue
Block a user