[MooncakeStore] Re-derive full external hits on stored boundaries (#49481)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Dao007forever
2026-07-23 18:12:28 +08:00
committed by GitHub
co-authored by Yifan Qiao Claude
parent a4904ba903
commit a76df87db8
8 changed files with 110 additions and 38 deletions
@@ -410,7 +410,7 @@ def test_lookup_key_client_lookup_prepends_typed_tag():
# Blocking lookup (non_block defaults to False) runs on the executor and
# returns the resolved hit length.
assert client.lookup("req0", token_len=128, block_hashes=[]) == 5
assert client.lookup("req0", num_tokens=128, block_hashes=[]) == 5
sent_frames = fake_socket.send_multipart.call_args[0][0]
assert sent_frames[0] == protocol.LOOKUP_MSG
@@ -439,11 +439,11 @@ def test_lookup_key_client_reset_uses_typed_protocol():
assert client.reset() is False
def _poll_lookup(client, req_id, token_len=128, block_hashes=(), timeout=5.0):
def _poll_lookup(client, req_id, num_tokens=128, block_hashes=(), timeout=5.0):
"""Drive non-blocking lookup until the executor completes it."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
result = client.lookup(req_id, token_len, list(block_hashes), non_block=True)
result = client.lookup(req_id, num_tokens, list(block_hashes), non_block=True)
if result is not None:
return result
time.sleep(0.005)
@@ -240,7 +240,10 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
worker.store = store
# Both groups stored all 4 blocks -> full hit.
assert worker.lookup(token_len=64, block_hashes=hs) == 64
assert worker.lookup(num_tokens=65, block_hashes=hs) == 64
# Exact-multiple prompt: the full hit is re-derived one block lower,
# where both groups' stored blocks still cover the SWA window.
assert worker.lookup(num_tokens=64, block_hashes=hs) == 48
# Evict SWA's first two blocks (outside its window of 32 tokens = 2 blocks).
swa_keys_outside_window = [
@@ -253,7 +256,12 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
# SWA window=32 -> only last 2 blocks must be present in SWA group.
# Full has all 4. Coordinator should still return 64.
assert worker.lookup(token_len=64, block_hashes=hs) == 64
assert worker.lookup(num_tokens=65, block_hashes=hs) == 64
# Exact-multiple prompt after eviction: the boundary one block lower
# needs SWA block 1, which is gone — no usable stored boundary remains
# (the pre-fix arithmetic clamp would have returned 48 and livelocked
# on load failure -> recompute -> same lookup).
assert worker.lookup(num_tokens=64, block_hashes=hs) == 0
def test_recv_skips_swa_blocks_before_window():
@@ -473,27 +473,25 @@ def test_from_request_tracker_no_load_saves_normally():
class _StubLookupClient:
def __init__(self, hit_tokens: int) -> None:
self._hit_tokens = hit_tokens
self.num_tokens: list[int] = []
def lookup(
self,
req_id: str,
token_len: int,
num_tokens: int,
block_hashes: list[bytes],
non_block: bool = False,
) -> int:
self.num_tokens.append(num_tokens)
return self._hit_tokens
def test_full_external_hit_keeps_kvpool_cached_tokens_block_aligned():
# When the external store hits the entire prompt, scheduler must leave at
# least one token uncomputed for sampling but stay on a block boundary.
# Otherwise the recv-side load mask floors token_len to
# (num_tokens-1)//block_size, the tail partial chunk is dropped, and -- if
# the local cache covers the aligned prefix -- key_list ends up empty
# (ZeroDivisionError in the recv thread's `tp_rank % len(key_list)`).
# The worker re-derives a full external hit below the request end on an
# existing boundary, so the scheduler receives the usable aligned hit.
scheduler = _make_bare_scheduler()
scheduler.load_async = True
scheduler.client = _StubLookupClient(hit_tokens=48) # full hit on 48-token prompt
scheduler.client = _StubLookupClient(hit_tokens=32)
request = SimpleNamespace(
request_id="req-0",
@@ -510,6 +508,7 @@ def test_full_external_hit_keeps_kvpool_cached_tokens_block_aligned():
assert need_to_allocate == 16
assert load_async is True
load_spec = scheduler.load_specs["req-0"]
assert scheduler.client.num_tokens == [48]
assert load_spec.vllm_cached_tokens == 16
assert load_spec.kvpool_cached_tokens == 32
assert load_spec.kvpool_cached_tokens % 16 == 0
@@ -522,7 +521,7 @@ def test_full_external_hit_with_full_local_hit_skips_load():
# into any block-aligned key.
scheduler = _make_bare_scheduler()
scheduler.load_async = True
scheduler.client = _StubLookupClient(hit_tokens=48)
scheduler.client = _StubLookupClient(hit_tokens=32)
request = SimpleNamespace(
request_id="req-0",
@@ -1737,6 +1737,61 @@ def test_lookup_partial_prefix_returns_first_hit_length():
assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32
def test_lookup_full_hit_reuses_existing_boundary():
"""A full hit is re-derived below the request end without another RPC."""
worker = _make_bare_worker(block_size=16)
worker.store.batch_is_exist.return_value = [1, 1]
assert worker.lookup(32, [b"h0", b"h1"]) == 16
assert worker.store.batch_is_exist.call_count == 1
def test_lookup_full_hit_with_eagle_pops_once_not_twice():
"""Eagle already leaves the last block for the drafter, so a
full-prompt re-derivation must never fire for eagle-governed hits:
firing would anchor the search one block lower and pop a second
block, regressing the hit by an extra producer boundary."""
worker = _make_bare_worker(block_size=16)
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
worker._kv_cache_groups,
scheduler_block_size=16,
hash_block_size=16,
use_eagle=True,
)
worker.store.batch_is_exist.return_value = [1, 1, 1, 1]
# 64-token exact-multiple prompt, all 4 blocks stored: one eagle pop
# gives 48; a spurious re-derivation (anchored at 48) would pop again
# and return 32.
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 48
assert worker.store.batch_is_exist.call_count == 1
def test_lookup_full_hit_swa_degrades_when_no_stored_boundary_is_usable():
"""The motivating livelock: the producer of a 64-token prompt stored
only its SWA tail window (blocks 2-3). The old arithmetic clamp turned
the full hit into 48, whose SWA window needs the never-written block 1,
so every load failed and the recompute re-entered the same lookup. The
re-derivation must report that no stored boundary below the request end
is usable."""
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec
worker = _make_bare_worker(block_size=16)
swa = SlidingWindowSpec(
block_size=16, num_kv_heads=8, head_size=64, dtype=None, sliding_window=32
)
worker._kv_cache_groups = [KVCacheGroupSpec(["layer0"], swa)]
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
worker._kv_cache_groups,
scheduler_block_size=worker.hash_block_size,
hash_block_size=worker.hash_block_size,
)
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 0
assert worker.store.batch_is_exist.call_count == 1
def test_lookup_swa_single_group_returns_full_when_tail_window_present():
"""Single-SWA, sliding_window=32 (= 2 blocks): producer stored only the
tail. Coordinator-driven lookup returns full prefix even though the
@@ -1754,7 +1809,7 @@ def test_lookup_swa_single_group_returns_full_when_tail_window_present():
hash_block_size=worker.hash_block_size,
)
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 64
assert worker.lookup(65, [b"h0", b"h1", b"h2", b"h3"]) == 64
def test_lookup_checks_all_potential_swa_hit_boundaries():
@@ -2219,7 +2274,7 @@ def test_lookup_records_mooncake_metrics():
worker = _make_bare_worker()
worker.store.batch_is_exist.return_value = [1, 1]
result = worker.lookup(32, [b"a0", b"a1"])
result = worker.lookup(33, [b"a0", b"a1"])
stats = worker.get_kv_connector_stats()
assert result == 32
@@ -93,6 +93,9 @@ class MooncakeStoreCoordinator:
self.eagle_group_ids = set(range(len(kv_cache_groups)))
self._verify_and_split_kv_cache_groups()
def align_lookup_length(self, length: int) -> int:
return length // self.lcm_block_size * self.lcm_block_size
def _verify_and_split_kv_cache_groups(self) -> None:
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
dispatches via spec_manager_map (we don't allocate managers).
@@ -10,7 +10,8 @@ Wire format (REQ/REP over IPC):
Request: [msg_type: bytes] [payload_frames...]
msg_type == LOOKUP_MSG:
frame 1: token_len (u32 big-endian, 4 bytes)
frame 1: num_tokens (u32 big-endian, 4 bytes); the worker derives
the aligned lookup length
frame 2: hash_len (u16 big-endian, 2 bytes) — byte length of each
fixed-size block hash (0 when there are no hashes)
frame 3: raw block hashes concatenated back-to-back (each hash_len
@@ -80,14 +80,12 @@ class MooncakeStoreScheduler:
Returns ``(None, False)`` when an async lookup is still in flight,
signaling the scheduler to retry this request on a later step.
"""
# Look up against the full prefill range, not just the prompt.
token_len = request.num_tokens // self._block_size * self._block_size
if token_len < self._block_size:
if request.num_tokens < self._block_size:
return 0, False
num_external_hit_tokens = self.client.lookup(
request.request_id,
token_len,
request.num_tokens,
request.block_hashes,
non_block=self.lookup_async,
)
@@ -95,14 +93,6 @@ class MooncakeStoreScheduler:
# Lookup not ready yet; scheduler will retry on a later step.
return None, False
if num_external_hit_tokens == request.num_tokens:
# Leave a sub-block tail uncomputed for sampling, on a block
# boundary so the recv-side load mask covers every yielded chunk.
num_external_hit_tokens = max(
0,
(request.num_tokens - 1) // self._block_size * self._block_size,
)
if num_external_hit_tokens < num_computed_tokens:
need_to_allocate = 0
else:
@@ -1467,11 +1467,14 @@ class MooncakeStoreWorker:
return finished_sending
def lookup(self, token_len: int, block_hashes: Sequence[BlockHash]) -> int:
def lookup(self, num_tokens: int, block_hashes: Sequence[BlockHash]) -> int:
"""Check how many prefix tokens exist in the store.
Checks across all rank-specific key namespaces that may be loaded.
Checks across all rank-specific key namespaces that may be loaded. A
hit covering all ``num_tokens`` is re-derived below the request end so
the last token is recomputed for sampling.
"""
token_len = self.coord.align_lookup_length(num_tokens)
if not block_hashes or token_len <= 0:
return 0
@@ -1535,11 +1538,24 @@ class MooncakeStoreWorker:
)
}
cached_block_pool = ExternalCachedBlockPool(
self.hash_block_size,
exists_set,
)
_masks, hit_length = self.coord.find_longest_cache_hit(
block_hashes,
token_len,
ExternalCachedBlockPool(self.hash_block_size, exists_set),
cached_block_pool,
)
if hit_length >= num_tokens:
usable_length = self.coord.align_lookup_length(num_tokens - 1)
if usable_length <= 0:
return 0
_masks, hit_length = self.coord.find_longest_cache_hit(
block_hashes,
usable_length,
cached_block_pool,
)
return hit_length
def get_kv_events(self) -> list[BlockStored]:
@@ -1605,11 +1621,11 @@ class LookupKeyServer:
msg_type = bytes(all_frames[0])
if msg_type == LOOKUP_MSG:
token_len = int.from_bytes(all_frames[1], byteorder="big")
num_tokens = int.from_bytes(all_frames[1], byteorder="big")
hash_len = int.from_bytes(all_frames[2], byteorder="big")
blob = all_frames[3].buffer
block_hashes = BlobBlockHashes(blob, hash_len)
result = self.store_worker.lookup(token_len, block_hashes)
result = self.store_worker.lookup(num_tokens, block_hashes)
self.socket.send(result.to_bytes(4, "big"))
elif msg_type == RESET_MSG:
@@ -1672,11 +1688,11 @@ class LookupKeyClient:
)
self.futures: dict[str, Future[int]] = {}
def _lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int:
def _lookup(self, num_tokens: int, block_hashes: list[BlockHash]) -> int:
hash_len = len(block_hashes[0]) if block_hashes else 0
all_frames = (
LOOKUP_MSG,
token_len.to_bytes(4, byteorder="big"),
num_tokens.to_bytes(4, byteorder="big"),
hash_len.to_bytes(2, byteorder="big"),
b"".join(block_hashes),
)
@@ -1687,7 +1703,7 @@ class LookupKeyClient:
def lookup(
self,
req_id: str,
token_len: int,
num_tokens: int,
block_hashes: list[BlockHash],
non_block: bool = False,
) -> int | None:
@@ -1695,7 +1711,7 @@ class LookupKeyClient:
so the caller retries on a later step."""
future = self.futures.get(req_id)
if future is None:
future = self.executor.submit(self._lookup, token_len, list(block_hashes))
future = self.executor.submit(self._lookup, num_tokens, list(block_hashes))
self.futures[req_id] = future
if non_block and not future.done():
return None