[Bugfix][KV Offload] Reuse in-flight async lookup probes (#55823)

Signed-off-by: Alex <jihui.huang@daocloud.io>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
AlexHuang
2026-09-14 15:02:45 +03:00
committed by GitHub
co-authored by OpenAI Codex Or Ozeri
parent a5f6f61a8a
commit c612e2bff0
2 changed files with 112 additions and 21 deletions
@@ -8,7 +8,7 @@ from collections.abc import Iterable
import pytest
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, make_offload_key
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager, LookupPhase
def _key(i: int) -> OffloadKey:
@@ -46,10 +46,13 @@ class TestAsyncLookupManager:
def test_found_key_returns_true(self):
mgr = InMemoryLookupManager(existing_keys={_key(1)})
assert mgr.lookup(_key(1), _ctx()) is None
assert mgr._lookup_state[_key(1)].phase is LookupPhase.PENDING
mgr.flush()
assert mgr._lookup_state[_key(1)].phase is LookupPhase.IN_FLIGHT
mgr._results_ready.wait()
mgr._results_ready.clear()
assert mgr.lookup(_key(1), _ctx()) is True
assert mgr._lookup_state[_key(1)].phase is LookupPhase.RESOLVED
mgr.shutdown()
def test_not_found_key_returns_false(self):
@@ -106,6 +109,68 @@ class TestAsyncLookupManager:
assert _key(1) not in mgr._lookup_state
mgr.shutdown()
def test_cleanup_reuses_in_flight_probe(self, monkeypatch: pytest.MonkeyPatch):
"""A replacement request shares the in-flight probe and its verdict."""
key = _key(1)
mgr = InMemoryLookupManager(existing_keys={key})
ctx_b = _ctx("req_b")
probe_started = threading.Event()
release_probe = threading.Event()
batch_lookup = mgr.batch_lookup
def blocking_lookup(keys, req_context):
probe_started.set()
if not release_probe.wait(timeout=5):
raise TimeoutError("Test did not release the backend probe")
return batch_lookup(keys, req_context)
monkeypatch.setattr(mgr, "batch_lookup", blocking_lookup)
try:
assert mgr.lookup(key, _ctx("req_a")) is None
mgr.flush()
assert probe_started.wait(timeout=5)
assert mgr._lookup_state[key].phase is LookupPhase.IN_FLIGHT
mgr.cleanup("req_a")
assert mgr.lookup(key, ctx_b) is None
assert mgr._lookup_state[key].phase is LookupPhase.IN_FLIGHT
mgr.flush()
release_probe.set()
batch = mgr._pending_results.get(timeout=5)
mgr._pending_results.put(batch)
replacement_result = mgr.lookup(key, ctx_b)
finally:
release_probe.set()
mgr.shutdown()
assert mgr.batch_lookup_calls == 1
assert replacement_result is True
@pytest.mark.parametrize(
"reclaim_at_shutdown", [False, True], ids=["flush", "shutdown"]
)
def test_unclaimed_probe_reclaimed_without_lookup(self, reclaim_at_shutdown: bool):
"""A completed orphan is released by flush or shutdown without a lookup."""
key = _key(1)
mgr = InMemoryLookupManager(existing_keys={key})
try:
mgr.lookup(key, _ctx("req_a"))
mgr.flush()
mgr.cleanup("req_a")
assert key in mgr._lookup_state
assert not mgr._req_keys
if not reclaim_at_shutdown:
batch = mgr._pending_results.get(timeout=5)
mgr._pending_results.put(batch)
mgr.flush()
assert key not in mgr._lookup_state
finally:
mgr.shutdown()
assert not mgr._lookup_state
assert mgr._pending_results.empty()
def test_stale_result_ignored_after_cleanup_and_key_reuse(self):
key = _key(1)
mgr = InMemoryLookupManager()
@@ -119,11 +184,12 @@ class TestAsyncLookupManager:
generation = mgr._lookup_state[key].generation
assert generation != stale_generation
mgr.flush()
mgr._results_ready.wait()
mgr._results_ready.clear()
current_result = mgr._pending_results.get(timeout=5)
mgr._pending_results.put([(key, stale_generation, True)])
mgr.drain_results()
assert mgr.lookup(key, ctx_b) is None
mgr._pending_results.put([(key, generation, False)])
mgr._pending_results.put(current_result)
mgr.drain_results()
assert mgr.lookup(key, ctx_b) is False
mgr.shutdown()
+41 -16
View File
@@ -27,8 +27,10 @@ lookup() accumulates new keys in _lookup_batch without touching the queue.
flush() is called once per step from the tier's on_schedule_end(), posting
the entire batch as a single queue item so the background thread sees one
batch per step.
drain_results() is called before any lookup() calls in the same step, so
lookup() is a pure OrderedDict operation.
Results are drained on the first lookup after each flush, at flush(), and
after worker shutdown. In-flight lookups with no remaining request references
are retained until their results are drained, allowing new requests to share
the same probe.
"""
import queue
@@ -36,6 +38,7 @@ import threading
from abc import ABC, abstractmethod
from collections.abc import Collection, Iterable
from dataclasses import dataclass, field
from enum import Enum, auto
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
@@ -43,10 +46,21 @@ from vllm.v1.kv_offload.base import OffloadKey, ReqContext
logger = init_logger(__name__)
class LookupPhase(Enum):
"""Lifecycle phase of a lookup probe."""
PENDING = auto() # Accumulated in _lookup_batch, but not yet submitted.
IN_FLIGHT = auto() # Submitted to the worker, but not yet resolved.
RESOLVED = auto() # A final verdict has been applied to the state.
@dataclass(slots=True)
class LookupState:
generation: int
result: bool | None = None # True (found), False (not found), None
phase: LookupPhase = LookupPhase.PENDING
# None while pending/in flight; True if the key exists; False if absent or
# explicitly marked missing after a failed load.
result: bool | None = None
request_ids: set[str] = field(default_factory=set) # requests asking for the lookup
@@ -155,24 +169,28 @@ class AsyncLookupManager(ABC):
Called once per step from on_schedule_end() after all lookup() calls
are done. The worker receives the full batch and processes it during
the model-execution window, maximising time available before the next
step's drain_results(). Safe to call with an empty batch (no-op).
step's drain_results(). Also drains completed lookups when there
are no new keys to submit.
"""
self.drain_results()
self._need_to_drain = True
batch = self._lookup_batch
self._lookup_batch = []
batch = [
(key, req_context, generation)
for key, req_context, generation in batch
if (state := self._lookup_state.get(key)) is not None
and state.generation == generation
]
if batch:
self._lookup_queue.put(batch)
in_flight_batch = []
for key, req_context, generation in batch:
state = self._lookup_state.get(key)
if state is None or state.generation != generation:
continue
assert state.phase is LookupPhase.PENDING
state.phase = LookupPhase.IN_FLIGHT
in_flight_batch.append((key, req_context, generation))
if in_flight_batch:
self._lookup_queue.put(in_flight_batch)
def drain_results(self) -> None:
"""Apply pending worker results to _lookup_state.
Called from lookup() before checking state.
Called from lookup(), flush(), and shutdown() on the scheduler thread.
"""
while True:
try:
@@ -183,6 +201,10 @@ class AsyncLookupManager(ABC):
state = self._lookup_state.get(key)
if state is None or state.generation != generation:
continue
if not state.request_ids:
del self._lookup_state[key]
continue
assert state.phase is LookupPhase.IN_FLIGHT
# Each lookup generation is enqueued exactly once. A matching
# generation must not receive a second result; stale
# generations were discarded above.
@@ -192,6 +214,7 @@ class AsyncLookupManager(ABC):
"failed-load livelock"
)
state.result = result
state.phase = LookupPhase.RESOLVED
def mark_miss(self, keys: Collection[OffloadKey]) -> None:
"""Force the cached verdict for ``keys`` to False after a failed load, so
@@ -201,9 +224,10 @@ class AsyncLookupManager(ABC):
state = self._lookup_state.get(key)
if state is not None:
state.result = False
state.phase = LookupPhase.RESOLVED
def cleanup(self, req_id: str) -> None:
"""Remove entries no longer needed by any active request.
"""Release request references, retaining in-flight lookups.
Called from the tier's on_request_finished(). Uses the reverse
index to visit only keys associated with this request.
@@ -211,13 +235,14 @@ class AsyncLookupManager(ABC):
for key in self._req_keys.pop(req_id, ()):
state = self._lookup_state[key]
state.request_ids.discard(req_id)
if not state.request_ids:
if not state.request_ids and state.phase is not LookupPhase.IN_FLIGHT:
del self._lookup_state[key]
def shutdown(self) -> None:
"""Stop the worker thread."""
"""Stop the worker thread and drain completed lookups."""
self._lookup_queue.put(None) # unblock _worker from _lookup_queue.get()
self._thread.join()
self.drain_results()
# ------------------------------------------------------------------
# Internal helpers