[Bugfix][KV Offload] Ignore stale async lookup results (#54872)

Signed-off-by: Alex <jihui.huang@daocloud.io>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
AlexHuang
2026-09-02 17:10:26 +03:00
committed by GitHub
co-authored by OpenAI Codex
parent ffe3bb3c72
commit 35faf957d6
2 changed files with 56 additions and 27 deletions
@@ -104,6 +104,28 @@ class TestAsyncLookupManager:
assert _key(1) not in mgr._lookup_state assert _key(1) not in mgr._lookup_state
mgr.shutdown() mgr.shutdown()
def test_stale_result_ignored_after_cleanup_and_key_reuse(self):
key = _key(1)
mgr = InMemoryLookupManager()
ctx_a = _ctx("req_a")
ctx_b = _ctx("req_b")
assert mgr.lookup(key, ctx_a) is None
stale_generation = mgr._lookup_state[key].generation
mgr.cleanup("req_a")
assert mgr.lookup(key, ctx_b) is None
generation = mgr._lookup_state[key].generation
assert generation != stale_generation
mgr._pending_results.put([(key, stale_generation, True)])
mgr.drain_results()
assert mgr.lookup(key, ctx_b) is None
mgr._pending_results.put([(key, generation, False)])
mgr.drain_results()
assert mgr.lookup(key, ctx_b) is False
mgr.shutdown()
def test_flush_no_queue_post_when_empty(self): def test_flush_no_queue_post_when_empty(self):
mgr = InMemoryLookupManager() mgr = InMemoryLookupManager()
mgr.flush() mgr.flush()
@@ -220,7 +242,8 @@ class TestAsyncLookupManager:
# (b) A stray/duplicate result for the now-decided key violates the # (b) A stray/duplicate result for the now-decided key violates the
# enqueue-once invariant and must trip the assert. # enqueue-once invariant and must trip the assert.
mgr._pending_results.put([(_key(1), True)]) generation = mgr._lookup_state[_key(1)].generation
mgr._pending_results.put([(_key(1), generation, True)])
with pytest.raises(AssertionError): with pytest.raises(AssertionError):
mgr.drain_results() mgr.drain_results()
mgr.shutdown() mgr.shutdown()
+32 -26
View File
@@ -45,6 +45,7 @@ logger = init_logger(__name__)
@dataclass(slots=True) @dataclass(slots=True)
class LookupState: class LookupState:
generation: int
result: bool | None = None # True (found), False (not found), None result: bool | None = None # True (found), False (not found), None
request_ids: set[str] = field(default_factory=set) # requests asking for the lookup request_ids: set[str] = field(default_factory=set) # requests asking for the lookup
@@ -78,21 +79,24 @@ class AsyncLookupManager(ABC):
self._lookup_state: dict[OffloadKey, LookupState] = {} self._lookup_state: dict[OffloadKey, LookupState] = {}
# req_id → keys looked up by that request (reverse index for cleanup). # req_id → keys looked up by that request (reverse index for cleanup).
self._req_keys: dict[str, set[OffloadKey]] = {} self._req_keys: dict[str, set[OffloadKey]] = {}
# Results from an older state must not be applied after cleanup removes
# and recreates a key.
self._next_generation = 0
# Accumulates (key, req_context) pairs during lookup() calls. # Accumulates (key, req_context, generation) triples during lookup().
# Flushed as one queue item per step by flush(). # Flushed as one queue item per step by flush().
self._lookup_batch: list[tuple[OffloadKey, ReqContext]] = [] self._lookup_batch: list[tuple[OffloadKey, ReqContext, int]] = []
# Scheduler → worker: one full step's batch per item. # Scheduler → worker: one full step's batch per item.
# None is used as a shutdown sentinel. # None is used as a shutdown sentinel.
self._lookup_queue: queue.SimpleQueue[ self._lookup_queue: queue.SimpleQueue[
list[tuple[OffloadKey, ReqContext]] | None list[tuple[OffloadKey, ReqContext, int]] | None
] = queue.SimpleQueue() ] = queue.SimpleQueue()
# Worker → scheduler: completed result batches. # Worker → scheduler: completed result batches.
# Each item is a list of (key, found) pairs. # Each item is a list of (key, generation, found) triples.
# SimpleQueue is explicitly thread-safe for one writer / one reader. # SimpleQueue is explicitly thread-safe for one writer / one reader.
self._pending_results: queue.SimpleQueue[list[tuple[OffloadKey, bool]]] = ( self._pending_results: queue.SimpleQueue[list[tuple[OffloadKey, int, bool]]] = (
queue.SimpleQueue() queue.SimpleQueue()
) )
self._need_to_drain: bool = False self._need_to_drain: bool = False
@@ -137,9 +141,10 @@ class AsyncLookupManager(ABC):
req_id = req_context.req_id req_id = req_context.req_id
state = self._lookup_state.get(key) state = self._lookup_state.get(key)
if state is None: if state is None:
state = LookupState() state = LookupState(generation=self._next_generation)
self._next_generation += 1
self._lookup_state[key] = state self._lookup_state[key] = state
self._lookup_batch.append((key, req_context)) self._lookup_batch.append((key, req_context, state.generation))
state.request_ids.add(req_id) state.request_ids.add(req_id)
self._req_keys.setdefault(req_id, set()).add(key) self._req_keys.setdefault(req_id, set()).add(key)
return state.result return state.result
@@ -167,19 +172,19 @@ class AsyncLookupManager(ABC):
batch = self._pending_results.get_nowait() batch = self._pending_results.get_nowait()
except queue.Empty: except queue.Empty:
break break
for key, result in batch: for key, generation, result in batch:
state = self._lookup_state.get(key) state = self._lookup_state.get(key)
if state is not None: if state is None or state.generation != generation:
# A key is enqueued for probing exactly once, so a decided continue
# verdict must never receive a second result. Enforcing it # Each lookup generation is enqueued exactly once. A matching
# keeps a late/duplicate result from resurrecting a stale # generation must not receive a second result; stale
# True and reopening the failed-load livelock. # generations were discarded above.
assert state.result is None, ( assert state.result is None, (
"cached key received a second lookup result; the " "cached key received a second lookup result; the "
"enqueue-once invariant is broken and could reopen the " "enqueue-once invariant is broken and could reopen the "
"failed-load livelock" "failed-load livelock"
) )
state.result = result state.result = result
def mark_miss(self, keys: Collection[OffloadKey]) -> None: def mark_miss(self, keys: Collection[OffloadKey]) -> None:
"""Force the cached verdict for ``keys`` to False after a failed load, so """Force the cached verdict for ``keys`` to False after a failed load, so
@@ -218,18 +223,19 @@ class AsyncLookupManager(ABC):
break break
# Group by req_id. # Group by req_id.
batches: dict[str, tuple[ReqContext, list[OffloadKey]]] = {} batches: dict[str, tuple[ReqContext, list[tuple[OffloadKey, int]]]] = {}
for key, req_context in pending: for key, req_context, generation in pending:
req_id = req_context.req_id req_id = req_context.req_id
if req_id not in batches: if req_id not in batches:
batches[req_id] = (req_context, []) batches[req_id] = (req_context, [])
batches[req_id][1].append(key) batches[req_id][1].append((key, generation))
if not batches: if not batches:
continue continue
results: list[tuple[OffloadKey, bool]] = [] results: list[tuple[OffloadKey, int, bool]] = []
for req_context, keys in batches.values(): for req_context, entries in batches.values():
keys = [key for key, _ in entries]
try: try:
hits = self.batch_lookup(keys, req_context) hits = self.batch_lookup(keys, req_context)
except Exception as exc: except Exception as exc:
@@ -241,8 +247,8 @@ class AsyncLookupManager(ABC):
) )
hits = (False for _ in keys) hits = (False for _ in keys)
for key, hit in zip(keys, hits): for (key, generation), hit in zip(entries, hits):
results.append((key, hit)) results.append((key, generation, hit))
# Post the entire batch as one item — no lock needed. # Post the entire batch as one item — no lock needed.
if results: if results: