[Bugfix][KV Offload] Reuse in-flight async lookup probes (#55823)

Signed-off-by: Alex <jihui.huang@daocloud.io>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
AlexHuang
2026-09-14 15:02:45 +03:00
committed by GitHub
co-authored by OpenAI Codex Or Ozeri
parent a5f6f61a8a
commit c612e2bff0
2 changed files with 112 additions and 21 deletions
@@ -8,7 +8,7 @@ from collections.abc import Iterable
import pytest
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, make_offload_key
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager, LookupPhase
def _key(i: int) -> OffloadKey:
@@ -46,10 +46,13 @@ class TestAsyncLookupManager:
def test_found_key_returns_true(self):
mgr = InMemoryLookupManager(existing_keys={_key(1)})
assert mgr.lookup(_key(1), _ctx()) is None
assert mgr._lookup_state[_key(1)].phase is LookupPhase.PENDING
mgr.flush()
assert mgr._lookup_state[_key(1)].phase is LookupPhase.IN_FLIGHT
mgr._results_ready.wait()
mgr._results_ready.clear()
assert mgr.lookup(_key(1), _ctx()) is True
assert mgr._lookup_state[_key(1)].phase is LookupPhase.RESOLVED
mgr.shutdown()
def test_not_found_key_returns_false(self):
@@ -106,6 +109,68 @@ class TestAsyncLookupManager:
assert _key(1) not in mgr._lookup_state
mgr.shutdown()
def test_cleanup_reuses_in_flight_probe(self, monkeypatch: pytest.MonkeyPatch):
"""A replacement request shares the in-flight probe and its verdict."""
key = _key(1)
mgr = InMemoryLookupManager(existing_keys={key})
ctx_b = _ctx("req_b")
probe_started = threading.Event()
release_probe = threading.Event()
batch_lookup = mgr.batch_lookup
def blocking_lookup(keys, req_context):
probe_started.set()
if not release_probe.wait(timeout=5):
raise TimeoutError("Test did not release the backend probe")
return batch_lookup(keys, req_context)
monkeypatch.setattr(mgr, "batch_lookup", blocking_lookup)
try:
assert mgr.lookup(key, _ctx("req_a")) is None
mgr.flush()
assert probe_started.wait(timeout=5)
assert mgr._lookup_state[key].phase is LookupPhase.IN_FLIGHT
mgr.cleanup("req_a")
assert mgr.lookup(key, ctx_b) is None
assert mgr._lookup_state[key].phase is LookupPhase.IN_FLIGHT
mgr.flush()
release_probe.set()
batch = mgr._pending_results.get(timeout=5)
mgr._pending_results.put(batch)
replacement_result = mgr.lookup(key, ctx_b)
finally:
release_probe.set()
mgr.shutdown()
assert mgr.batch_lookup_calls == 1
assert replacement_result is True
@pytest.mark.parametrize(
"reclaim_at_shutdown", [False, True], ids=["flush", "shutdown"]
)
def test_unclaimed_probe_reclaimed_without_lookup(self, reclaim_at_shutdown: bool):
"""A completed orphan is released by flush or shutdown without a lookup."""
key = _key(1)
mgr = InMemoryLookupManager(existing_keys={key})
try:
mgr.lookup(key, _ctx("req_a"))
mgr.flush()
mgr.cleanup("req_a")
assert key in mgr._lookup_state
assert not mgr._req_keys
if not reclaim_at_shutdown:
batch = mgr._pending_results.get(timeout=5)
mgr._pending_results.put(batch)
mgr.flush()
assert key not in mgr._lookup_state
finally:
mgr.shutdown()
assert not mgr._lookup_state
assert mgr._pending_results.empty()
def test_stale_result_ignored_after_cleanup_and_key_reuse(self):
key = _key(1)
mgr = InMemoryLookupManager()
@@ -119,11 +184,12 @@ class TestAsyncLookupManager:
generation = mgr._lookup_state[key].generation
assert generation != stale_generation
mgr.flush()
mgr._results_ready.wait()
mgr._results_ready.clear()
current_result = mgr._pending_results.get(timeout=5)
mgr._pending_results.put([(key, stale_generation, True)])
mgr.drain_results()
assert mgr.lookup(key, ctx_b) is None
mgr._pending_results.put([(key, generation, False)])
mgr._pending_results.put(current_result)
mgr.drain_results()
assert mgr.lookup(key, ctx_b) is False
mgr.shutdown()