[MooncakeStore] Re-derive full external hits on stored boundaries (#49481)
Signed-off-by: Dao Le <Dao007forever@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Yifan Qiao
Claude
parent
a4904ba903
commit
a76df87db8
@@ -410,7 +410,7 @@ def test_lookup_key_client_lookup_prepends_typed_tag():
|
||||
|
||||
# Blocking lookup (non_block defaults to False) runs on the executor and
|
||||
# returns the resolved hit length.
|
||||
assert client.lookup("req0", token_len=128, block_hashes=[]) == 5
|
||||
assert client.lookup("req0", num_tokens=128, block_hashes=[]) == 5
|
||||
|
||||
sent_frames = fake_socket.send_multipart.call_args[0][0]
|
||||
assert sent_frames[0] == protocol.LOOKUP_MSG
|
||||
@@ -439,11 +439,11 @@ def test_lookup_key_client_reset_uses_typed_protocol():
|
||||
assert client.reset() is False
|
||||
|
||||
|
||||
def _poll_lookup(client, req_id, token_len=128, block_hashes=(), timeout=5.0):
|
||||
def _poll_lookup(client, req_id, num_tokens=128, block_hashes=(), timeout=5.0):
|
||||
"""Drive non-blocking lookup until the executor completes it."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = client.lookup(req_id, token_len, list(block_hashes), non_block=True)
|
||||
result = client.lookup(req_id, num_tokens, list(block_hashes), non_block=True)
|
||||
if result is not None:
|
||||
return result
|
||||
time.sleep(0.005)
|
||||
|
||||
@@ -240,7 +240,10 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
worker.store = store
|
||||
|
||||
# Both groups stored all 4 blocks -> full hit.
|
||||
assert worker.lookup(token_len=64, block_hashes=hs) == 64
|
||||
assert worker.lookup(num_tokens=65, block_hashes=hs) == 64
|
||||
# Exact-multiple prompt: the full hit is re-derived one block lower,
|
||||
# where both groups' stored blocks still cover the SWA window.
|
||||
assert worker.lookup(num_tokens=64, block_hashes=hs) == 48
|
||||
|
||||
# Evict SWA's first two blocks (outside its window of 32 tokens = 2 blocks).
|
||||
swa_keys_outside_window = [
|
||||
@@ -253,7 +256,12 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
|
||||
# SWA window=32 -> only last 2 blocks must be present in SWA group.
|
||||
# Full has all 4. Coordinator should still return 64.
|
||||
assert worker.lookup(token_len=64, block_hashes=hs) == 64
|
||||
assert worker.lookup(num_tokens=65, block_hashes=hs) == 64
|
||||
# Exact-multiple prompt after eviction: the boundary one block lower
|
||||
# needs SWA block 1, which is gone — no usable stored boundary remains
|
||||
# (the pre-fix arithmetic clamp would have returned 48 and livelocked
|
||||
# on load failure -> recompute -> same lookup).
|
||||
assert worker.lookup(num_tokens=64, block_hashes=hs) == 0
|
||||
|
||||
|
||||
def test_recv_skips_swa_blocks_before_window():
|
||||
|
||||
@@ -473,27 +473,25 @@ def test_from_request_tracker_no_load_saves_normally():
|
||||
class _StubLookupClient:
|
||||
def __init__(self, hit_tokens: int) -> None:
|
||||
self._hit_tokens = hit_tokens
|
||||
self.num_tokens: list[int] = []
|
||||
|
||||
def lookup(
|
||||
self,
|
||||
req_id: str,
|
||||
token_len: int,
|
||||
num_tokens: int,
|
||||
block_hashes: list[bytes],
|
||||
non_block: bool = False,
|
||||
) -> int:
|
||||
self.num_tokens.append(num_tokens)
|
||||
return self._hit_tokens
|
||||
|
||||
|
||||
def test_full_external_hit_keeps_kvpool_cached_tokens_block_aligned():
|
||||
# When the external store hits the entire prompt, scheduler must leave at
|
||||
# least one token uncomputed for sampling but stay on a block boundary.
|
||||
# Otherwise the recv-side load mask floors token_len to
|
||||
# (num_tokens-1)//block_size, the tail partial chunk is dropped, and -- if
|
||||
# the local cache covers the aligned prefix -- key_list ends up empty
|
||||
# (ZeroDivisionError in the recv thread's `tp_rank % len(key_list)`).
|
||||
# The worker re-derives a full external hit below the request end on an
|
||||
# existing boundary, so the scheduler receives the usable aligned hit.
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler.load_async = True
|
||||
scheduler.client = _StubLookupClient(hit_tokens=48) # full hit on 48-token prompt
|
||||
scheduler.client = _StubLookupClient(hit_tokens=32)
|
||||
|
||||
request = SimpleNamespace(
|
||||
request_id="req-0",
|
||||
@@ -510,6 +508,7 @@ def test_full_external_hit_keeps_kvpool_cached_tokens_block_aligned():
|
||||
assert need_to_allocate == 16
|
||||
assert load_async is True
|
||||
load_spec = scheduler.load_specs["req-0"]
|
||||
assert scheduler.client.num_tokens == [48]
|
||||
assert load_spec.vllm_cached_tokens == 16
|
||||
assert load_spec.kvpool_cached_tokens == 32
|
||||
assert load_spec.kvpool_cached_tokens % 16 == 0
|
||||
@@ -522,7 +521,7 @@ def test_full_external_hit_with_full_local_hit_skips_load():
|
||||
# into any block-aligned key.
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler.load_async = True
|
||||
scheduler.client = _StubLookupClient(hit_tokens=48)
|
||||
scheduler.client = _StubLookupClient(hit_tokens=32)
|
||||
|
||||
request = SimpleNamespace(
|
||||
request_id="req-0",
|
||||
|
||||
@@ -1737,6 +1737,61 @@ def test_lookup_partial_prefix_returns_first_hit_length():
|
||||
assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32
|
||||
|
||||
|
||||
def test_lookup_full_hit_reuses_existing_boundary():
|
||||
"""A full hit is re-derived below the request end without another RPC."""
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
worker.store.batch_is_exist.return_value = [1, 1]
|
||||
|
||||
assert worker.lookup(32, [b"h0", b"h1"]) == 16
|
||||
assert worker.store.batch_is_exist.call_count == 1
|
||||
|
||||
|
||||
def test_lookup_full_hit_with_eagle_pops_once_not_twice():
|
||||
"""Eagle already leaves the last block for the drafter, so a
|
||||
full-prompt re-derivation must never fire for eagle-governed hits:
|
||||
firing would anchor the search one block lower and pop a second
|
||||
block, regressing the hit by an extra producer boundary."""
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=16,
|
||||
use_eagle=True,
|
||||
)
|
||||
worker.store.batch_is_exist.return_value = [1, 1, 1, 1]
|
||||
|
||||
# 64-token exact-multiple prompt, all 4 blocks stored: one eagle pop
|
||||
# gives 48; a spurious re-derivation (anchored at 48) would pop again
|
||||
# and return 32.
|
||||
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 48
|
||||
assert worker.store.batch_is_exist.call_count == 1
|
||||
|
||||
|
||||
def test_lookup_full_hit_swa_degrades_when_no_stored_boundary_is_usable():
|
||||
"""The motivating livelock: the producer of a 64-token prompt stored
|
||||
only its SWA tail window (blocks 2-3). The old arithmetic clamp turned
|
||||
the full hit into 48, whose SWA window needs the never-written block 1,
|
||||
so every load failed and the recompute re-entered the same lookup. The
|
||||
re-derivation must report that no stored boundary below the request end
|
||||
is usable."""
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec
|
||||
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
swa = SlidingWindowSpec(
|
||||
block_size=16, num_kv_heads=8, head_size=64, dtype=None, sliding_window=32
|
||||
)
|
||||
worker._kv_cache_groups = [KVCacheGroupSpec(["layer0"], swa)]
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
scheduler_block_size=worker.hash_block_size,
|
||||
hash_block_size=worker.hash_block_size,
|
||||
)
|
||||
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
|
||||
|
||||
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 0
|
||||
assert worker.store.batch_is_exist.call_count == 1
|
||||
|
||||
|
||||
def test_lookup_swa_single_group_returns_full_when_tail_window_present():
|
||||
"""Single-SWA, sliding_window=32 (= 2 blocks): producer stored only the
|
||||
tail. Coordinator-driven lookup returns full prefix even though the
|
||||
@@ -1754,7 +1809,7 @@ def test_lookup_swa_single_group_returns_full_when_tail_window_present():
|
||||
hash_block_size=worker.hash_block_size,
|
||||
)
|
||||
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
|
||||
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 64
|
||||
assert worker.lookup(65, [b"h0", b"h1", b"h2", b"h3"]) == 64
|
||||
|
||||
|
||||
def test_lookup_checks_all_potential_swa_hit_boundaries():
|
||||
@@ -2219,7 +2274,7 @@ def test_lookup_records_mooncake_metrics():
|
||||
worker = _make_bare_worker()
|
||||
worker.store.batch_is_exist.return_value = [1, 1]
|
||||
|
||||
result = worker.lookup(32, [b"a0", b"a1"])
|
||||
result = worker.lookup(33, [b"a0", b"a1"])
|
||||
stats = worker.get_kv_connector_stats()
|
||||
|
||||
assert result == 32
|
||||
|
||||
Reference in New Issue
Block a user