[MooncakeStore] Re-derive full external hits on stored boundaries (#49481)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Dao007forever
2026-07-23 18:12:28 +08:00
committed by GitHub
co-authored by Yifan Qiao Claude
parent a4904ba903
commit a76df87db8
8 changed files with 110 additions and 38 deletions
@@ -410,7 +410,7 @@ def test_lookup_key_client_lookup_prepends_typed_tag():
# Blocking lookup (non_block defaults to False) runs on the executor and
# returns the resolved hit length.
assert client.lookup("req0", token_len=128, block_hashes=[]) == 5
assert client.lookup("req0", num_tokens=128, block_hashes=[]) == 5
sent_frames = fake_socket.send_multipart.call_args[0][0]
assert sent_frames[0] == protocol.LOOKUP_MSG
@@ -439,11 +439,11 @@ def test_lookup_key_client_reset_uses_typed_protocol():
assert client.reset() is False
def _poll_lookup(client, req_id, token_len=128, block_hashes=(), timeout=5.0):
def _poll_lookup(client, req_id, num_tokens=128, block_hashes=(), timeout=5.0):
"""Drive non-blocking lookup until the executor completes it."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
result = client.lookup(req_id, token_len, list(block_hashes), non_block=True)
result = client.lookup(req_id, num_tokens, list(block_hashes), non_block=True)
if result is not None:
return result
time.sleep(0.005)
@@ -240,7 +240,10 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
worker.store = store
# Both groups stored all 4 blocks -> full hit.
assert worker.lookup(token_len=64, block_hashes=hs) == 64
assert worker.lookup(num_tokens=65, block_hashes=hs) == 64
# Exact-multiple prompt: the full hit is re-derived one block lower,
# where both groups' stored blocks still cover the SWA window.
assert worker.lookup(num_tokens=64, block_hashes=hs) == 48
# Evict SWA's first two blocks (outside its window of 32 tokens = 2 blocks).
swa_keys_outside_window = [
@@ -253,7 +256,12 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
# SWA window=32 -> only last 2 blocks must be present in SWA group.
# Full has all 4. Coordinator should still return 64.
assert worker.lookup(token_len=64, block_hashes=hs) == 64
assert worker.lookup(num_tokens=65, block_hashes=hs) == 64
# Exact-multiple prompt after eviction: the boundary one block lower
# needs SWA block 1, which is gone — no usable stored boundary remains
# (the pre-fix arithmetic clamp would have returned 48 and livelocked
# on load failure -> recompute -> same lookup).
assert worker.lookup(num_tokens=64, block_hashes=hs) == 0
def test_recv_skips_swa_blocks_before_window():
@@ -473,27 +473,25 @@ def test_from_request_tracker_no_load_saves_normally():
class _StubLookupClient:
def __init__(self, hit_tokens: int) -> None:
self._hit_tokens = hit_tokens
self.num_tokens: list[int] = []
def lookup(
self,
req_id: str,
token_len: int,
num_tokens: int,
block_hashes: list[bytes],
non_block: bool = False,
) -> int:
self.num_tokens.append(num_tokens)
return self._hit_tokens
def test_full_external_hit_keeps_kvpool_cached_tokens_block_aligned():
# When the external store hits the entire prompt, scheduler must leave at
# least one token uncomputed for sampling but stay on a block boundary.
# Otherwise the recv-side load mask floors token_len to
# (num_tokens-1)//block_size, the tail partial chunk is dropped, and -- if
# the local cache covers the aligned prefix -- key_list ends up empty
# (ZeroDivisionError in the recv thread's `tp_rank % len(key_list)`).
# The worker re-derives a full external hit below the request end on an
# existing boundary, so the scheduler receives the usable aligned hit.
scheduler = _make_bare_scheduler()
scheduler.load_async = True
scheduler.client = _StubLookupClient(hit_tokens=48) # full hit on 48-token prompt
scheduler.client = _StubLookupClient(hit_tokens=32)
request = SimpleNamespace(
request_id="req-0",
@@ -510,6 +508,7 @@ def test_full_external_hit_keeps_kvpool_cached_tokens_block_aligned():
assert need_to_allocate == 16
assert load_async is True
load_spec = scheduler.load_specs["req-0"]
assert scheduler.client.num_tokens == [48]
assert load_spec.vllm_cached_tokens == 16
assert load_spec.kvpool_cached_tokens == 32
assert load_spec.kvpool_cached_tokens % 16 == 0
@@ -522,7 +521,7 @@ def test_full_external_hit_with_full_local_hit_skips_load():
# into any block-aligned key.
scheduler = _make_bare_scheduler()
scheduler.load_async = True
scheduler.client = _StubLookupClient(hit_tokens=48)
scheduler.client = _StubLookupClient(hit_tokens=32)
request = SimpleNamespace(
request_id="req-0",
@@ -1737,6 +1737,61 @@ def test_lookup_partial_prefix_returns_first_hit_length():
assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32
def test_lookup_full_hit_reuses_existing_boundary():
"""A full hit is re-derived below the request end without another RPC."""
worker = _make_bare_worker(block_size=16)
worker.store.batch_is_exist.return_value = [1, 1]
assert worker.lookup(32, [b"h0", b"h1"]) == 16
assert worker.store.batch_is_exist.call_count == 1
def test_lookup_full_hit_with_eagle_pops_once_not_twice():
"""Eagle already leaves the last block for the drafter, so a
full-prompt re-derivation must never fire for eagle-governed hits:
firing would anchor the search one block lower and pop a second
block, regressing the hit by an extra producer boundary."""
worker = _make_bare_worker(block_size=16)
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
worker._kv_cache_groups,
scheduler_block_size=16,
hash_block_size=16,
use_eagle=True,
)
worker.store.batch_is_exist.return_value = [1, 1, 1, 1]
# 64-token exact-multiple prompt, all 4 blocks stored: one eagle pop
# gives 48; a spurious re-derivation (anchored at 48) would pop again
# and return 32.
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 48
assert worker.store.batch_is_exist.call_count == 1
def test_lookup_full_hit_swa_degrades_when_no_stored_boundary_is_usable():
"""The motivating livelock: the producer of a 64-token prompt stored
only its SWA tail window (blocks 2-3). The old arithmetic clamp turned
the full hit into 48, whose SWA window needs the never-written block 1,
so every load failed and the recompute re-entered the same lookup. The
re-derivation must report that no stored boundary below the request end
is usable."""
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec
worker = _make_bare_worker(block_size=16)
swa = SlidingWindowSpec(
block_size=16, num_kv_heads=8, head_size=64, dtype=None, sliding_window=32
)
worker._kv_cache_groups = [KVCacheGroupSpec(["layer0"], swa)]
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
worker._kv_cache_groups,
scheduler_block_size=worker.hash_block_size,
hash_block_size=worker.hash_block_size,
)
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 0
assert worker.store.batch_is_exist.call_count == 1
def test_lookup_swa_single_group_returns_full_when_tail_window_present():
"""Single-SWA, sliding_window=32 (= 2 blocks): producer stored only the
tail. Coordinator-driven lookup returns full prefix even though the
@@ -1754,7 +1809,7 @@ def test_lookup_swa_single_group_returns_full_when_tail_window_present():
hash_block_size=worker.hash_block_size,
)
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 64
assert worker.lookup(65, [b"h0", b"h1", b"h2", b"h3"]) == 64
def test_lookup_checks_all_potential_swa_hit_boundaries():
@@ -2219,7 +2274,7 @@ def test_lookup_records_mooncake_metrics():
worker = _make_bare_worker()
worker.store.batch_is_exist.return_value = [1, 1]
result = worker.lookup(32, [b"a0", b"a1"])
result = worker.lookup(33, [b"a0", b"a1"])
stats = worker.get_kv_connector_stats()
assert result == 32