[MooncakeStore] Re-derive full external hits on stored boundaries (#49481)
Signed-off-by: Dao Le <Dao007forever@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Yifan Qiao
Claude
parent
a4904ba903
commit
a76df87db8
@@ -410,7 +410,7 @@ def test_lookup_key_client_lookup_prepends_typed_tag():
|
||||
|
||||
# Blocking lookup (non_block defaults to False) runs on the executor and
|
||||
# returns the resolved hit length.
|
||||
assert client.lookup("req0", token_len=128, block_hashes=[]) == 5
|
||||
assert client.lookup("req0", num_tokens=128, block_hashes=[]) == 5
|
||||
|
||||
sent_frames = fake_socket.send_multipart.call_args[0][0]
|
||||
assert sent_frames[0] == protocol.LOOKUP_MSG
|
||||
@@ -439,11 +439,11 @@ def test_lookup_key_client_reset_uses_typed_protocol():
|
||||
assert client.reset() is False
|
||||
|
||||
|
||||
def _poll_lookup(client, req_id, token_len=128, block_hashes=(), timeout=5.0):
|
||||
def _poll_lookup(client, req_id, num_tokens=128, block_hashes=(), timeout=5.0):
|
||||
"""Drive non-blocking lookup until the executor completes it."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = client.lookup(req_id, token_len, list(block_hashes), non_block=True)
|
||||
result = client.lookup(req_id, num_tokens, list(block_hashes), non_block=True)
|
||||
if result is not None:
|
||||
return result
|
||||
time.sleep(0.005)
|
||||
|
||||
@@ -240,7 +240,10 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
worker.store = store
|
||||
|
||||
# Both groups stored all 4 blocks -> full hit.
|
||||
assert worker.lookup(token_len=64, block_hashes=hs) == 64
|
||||
assert worker.lookup(num_tokens=65, block_hashes=hs) == 64
|
||||
# Exact-multiple prompt: the full hit is re-derived one block lower,
|
||||
# where both groups' stored blocks still cover the SWA window.
|
||||
assert worker.lookup(num_tokens=64, block_hashes=hs) == 48
|
||||
|
||||
# Evict SWA's first two blocks (outside its window of 32 tokens = 2 blocks).
|
||||
swa_keys_outside_window = [
|
||||
@@ -253,7 +256,12 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
|
||||
# SWA window=32 -> only last 2 blocks must be present in SWA group.
|
||||
# Full has all 4. Coordinator should still return 64.
|
||||
assert worker.lookup(token_len=64, block_hashes=hs) == 64
|
||||
assert worker.lookup(num_tokens=65, block_hashes=hs) == 64
|
||||
# Exact-multiple prompt after eviction: the boundary one block lower
|
||||
# needs SWA block 1, which is gone — no usable stored boundary remains
|
||||
# (the pre-fix arithmetic clamp would have returned 48 and livelocked
|
||||
# on load failure -> recompute -> same lookup).
|
||||
assert worker.lookup(num_tokens=64, block_hashes=hs) == 0
|
||||
|
||||
|
||||
def test_recv_skips_swa_blocks_before_window():
|
||||
|
||||
@@ -473,27 +473,25 @@ def test_from_request_tracker_no_load_saves_normally():
|
||||
class _StubLookupClient:
|
||||
def __init__(self, hit_tokens: int) -> None:
|
||||
self._hit_tokens = hit_tokens
|
||||
self.num_tokens: list[int] = []
|
||||
|
||||
def lookup(
|
||||
self,
|
||||
req_id: str,
|
||||
token_len: int,
|
||||
num_tokens: int,
|
||||
block_hashes: list[bytes],
|
||||
non_block: bool = False,
|
||||
) -> int:
|
||||
self.num_tokens.append(num_tokens)
|
||||
return self._hit_tokens
|
||||
|
||||
|
||||
def test_full_external_hit_keeps_kvpool_cached_tokens_block_aligned():
|
||||
# When the external store hits the entire prompt, scheduler must leave at
|
||||
# least one token uncomputed for sampling but stay on a block boundary.
|
||||
# Otherwise the recv-side load mask floors token_len to
|
||||
# (num_tokens-1)//block_size, the tail partial chunk is dropped, and -- if
|
||||
# the local cache covers the aligned prefix -- key_list ends up empty
|
||||
# (ZeroDivisionError in the recv thread's `tp_rank % len(key_list)`).
|
||||
# The worker re-derives a full external hit below the request end on an
|
||||
# existing boundary, so the scheduler receives the usable aligned hit.
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler.load_async = True
|
||||
scheduler.client = _StubLookupClient(hit_tokens=48) # full hit on 48-token prompt
|
||||
scheduler.client = _StubLookupClient(hit_tokens=32)
|
||||
|
||||
request = SimpleNamespace(
|
||||
request_id="req-0",
|
||||
@@ -510,6 +508,7 @@ def test_full_external_hit_keeps_kvpool_cached_tokens_block_aligned():
|
||||
assert need_to_allocate == 16
|
||||
assert load_async is True
|
||||
load_spec = scheduler.load_specs["req-0"]
|
||||
assert scheduler.client.num_tokens == [48]
|
||||
assert load_spec.vllm_cached_tokens == 16
|
||||
assert load_spec.kvpool_cached_tokens == 32
|
||||
assert load_spec.kvpool_cached_tokens % 16 == 0
|
||||
@@ -522,7 +521,7 @@ def test_full_external_hit_with_full_local_hit_skips_load():
|
||||
# into any block-aligned key.
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler.load_async = True
|
||||
scheduler.client = _StubLookupClient(hit_tokens=48)
|
||||
scheduler.client = _StubLookupClient(hit_tokens=32)
|
||||
|
||||
request = SimpleNamespace(
|
||||
request_id="req-0",
|
||||
|
||||
@@ -1737,6 +1737,61 @@ def test_lookup_partial_prefix_returns_first_hit_length():
|
||||
assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32
|
||||
|
||||
|
||||
def test_lookup_full_hit_reuses_existing_boundary():
|
||||
"""A full hit is re-derived below the request end without another RPC."""
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
worker.store.batch_is_exist.return_value = [1, 1]
|
||||
|
||||
assert worker.lookup(32, [b"h0", b"h1"]) == 16
|
||||
assert worker.store.batch_is_exist.call_count == 1
|
||||
|
||||
|
||||
def test_lookup_full_hit_with_eagle_pops_once_not_twice():
|
||||
"""Eagle already leaves the last block for the drafter, so a
|
||||
full-prompt re-derivation must never fire for eagle-governed hits:
|
||||
firing would anchor the search one block lower and pop a second
|
||||
block, regressing the hit by an extra producer boundary."""
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=16,
|
||||
use_eagle=True,
|
||||
)
|
||||
worker.store.batch_is_exist.return_value = [1, 1, 1, 1]
|
||||
|
||||
# 64-token exact-multiple prompt, all 4 blocks stored: one eagle pop
|
||||
# gives 48; a spurious re-derivation (anchored at 48) would pop again
|
||||
# and return 32.
|
||||
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 48
|
||||
assert worker.store.batch_is_exist.call_count == 1
|
||||
|
||||
|
||||
def test_lookup_full_hit_swa_degrades_when_no_stored_boundary_is_usable():
|
||||
"""The motivating livelock: the producer of a 64-token prompt stored
|
||||
only its SWA tail window (blocks 2-3). The old arithmetic clamp turned
|
||||
the full hit into 48, whose SWA window needs the never-written block 1,
|
||||
so every load failed and the recompute re-entered the same lookup. The
|
||||
re-derivation must report that no stored boundary below the request end
|
||||
is usable."""
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec
|
||||
|
||||
worker = _make_bare_worker(block_size=16)
|
||||
swa = SlidingWindowSpec(
|
||||
block_size=16, num_kv_heads=8, head_size=64, dtype=None, sliding_window=32
|
||||
)
|
||||
worker._kv_cache_groups = [KVCacheGroupSpec(["layer0"], swa)]
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
scheduler_block_size=worker.hash_block_size,
|
||||
hash_block_size=worker.hash_block_size,
|
||||
)
|
||||
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
|
||||
|
||||
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 0
|
||||
assert worker.store.batch_is_exist.call_count == 1
|
||||
|
||||
|
||||
def test_lookup_swa_single_group_returns_full_when_tail_window_present():
|
||||
"""Single-SWA, sliding_window=32 (= 2 blocks): producer stored only the
|
||||
tail. Coordinator-driven lookup returns full prefix even though the
|
||||
@@ -1754,7 +1809,7 @@ def test_lookup_swa_single_group_returns_full_when_tail_window_present():
|
||||
hash_block_size=worker.hash_block_size,
|
||||
)
|
||||
worker.store.batch_is_exist.return_value = [0, 0, 1, 1]
|
||||
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 64
|
||||
assert worker.lookup(65, [b"h0", b"h1", b"h2", b"h3"]) == 64
|
||||
|
||||
|
||||
def test_lookup_checks_all_potential_swa_hit_boundaries():
|
||||
@@ -2219,7 +2274,7 @@ def test_lookup_records_mooncake_metrics():
|
||||
worker = _make_bare_worker()
|
||||
worker.store.batch_is_exist.return_value = [1, 1]
|
||||
|
||||
result = worker.lookup(32, [b"a0", b"a1"])
|
||||
result = worker.lookup(33, [b"a0", b"a1"])
|
||||
stats = worker.get_kv_connector_stats()
|
||||
|
||||
assert result == 32
|
||||
|
||||
@@ -93,6 +93,9 @@ class MooncakeStoreCoordinator:
|
||||
self.eagle_group_ids = set(range(len(kv_cache_groups)))
|
||||
self._verify_and_split_kv_cache_groups()
|
||||
|
||||
def align_lookup_length(self, length: int) -> int:
|
||||
return length // self.lcm_block_size * self.lcm_block_size
|
||||
|
||||
def _verify_and_split_kv_cache_groups(self) -> None:
|
||||
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
|
||||
dispatches via spec_manager_map (we don't allocate managers).
|
||||
|
||||
@@ -10,7 +10,8 @@ Wire format (REQ/REP over IPC):
|
||||
Request: [msg_type: bytes] [payload_frames...]
|
||||
|
||||
msg_type == LOOKUP_MSG:
|
||||
frame 1: token_len (u32 big-endian, 4 bytes)
|
||||
frame 1: num_tokens (u32 big-endian, 4 bytes); the worker derives
|
||||
the aligned lookup length
|
||||
frame 2: hash_len (u16 big-endian, 2 bytes) — byte length of each
|
||||
fixed-size block hash (0 when there are no hashes)
|
||||
frame 3: raw block hashes concatenated back-to-back (each hash_len
|
||||
|
||||
@@ -80,14 +80,12 @@ class MooncakeStoreScheduler:
|
||||
Returns ``(None, False)`` when an async lookup is still in flight,
|
||||
signaling the scheduler to retry this request on a later step.
|
||||
"""
|
||||
# Look up against the full prefill range, not just the prompt.
|
||||
token_len = request.num_tokens // self._block_size * self._block_size
|
||||
if token_len < self._block_size:
|
||||
if request.num_tokens < self._block_size:
|
||||
return 0, False
|
||||
|
||||
num_external_hit_tokens = self.client.lookup(
|
||||
request.request_id,
|
||||
token_len,
|
||||
request.num_tokens,
|
||||
request.block_hashes,
|
||||
non_block=self.lookup_async,
|
||||
)
|
||||
@@ -95,14 +93,6 @@ class MooncakeStoreScheduler:
|
||||
# Lookup not ready yet; scheduler will retry on a later step.
|
||||
return None, False
|
||||
|
||||
if num_external_hit_tokens == request.num_tokens:
|
||||
# Leave a sub-block tail uncomputed for sampling, on a block
|
||||
# boundary so the recv-side load mask covers every yielded chunk.
|
||||
num_external_hit_tokens = max(
|
||||
0,
|
||||
(request.num_tokens - 1) // self._block_size * self._block_size,
|
||||
)
|
||||
|
||||
if num_external_hit_tokens < num_computed_tokens:
|
||||
need_to_allocate = 0
|
||||
else:
|
||||
|
||||
@@ -1467,11 +1467,14 @@ class MooncakeStoreWorker:
|
||||
|
||||
return finished_sending
|
||||
|
||||
def lookup(self, token_len: int, block_hashes: Sequence[BlockHash]) -> int:
|
||||
def lookup(self, num_tokens: int, block_hashes: Sequence[BlockHash]) -> int:
|
||||
"""Check how many prefix tokens exist in the store.
|
||||
|
||||
Checks across all rank-specific key namespaces that may be loaded.
|
||||
Checks across all rank-specific key namespaces that may be loaded. A
|
||||
hit covering all ``num_tokens`` is re-derived below the request end so
|
||||
the last token is recomputed for sampling.
|
||||
"""
|
||||
token_len = self.coord.align_lookup_length(num_tokens)
|
||||
if not block_hashes or token_len <= 0:
|
||||
return 0
|
||||
|
||||
@@ -1535,11 +1538,24 @@ class MooncakeStoreWorker:
|
||||
)
|
||||
}
|
||||
|
||||
cached_block_pool = ExternalCachedBlockPool(
|
||||
self.hash_block_size,
|
||||
exists_set,
|
||||
)
|
||||
_masks, hit_length = self.coord.find_longest_cache_hit(
|
||||
block_hashes,
|
||||
token_len,
|
||||
ExternalCachedBlockPool(self.hash_block_size, exists_set),
|
||||
cached_block_pool,
|
||||
)
|
||||
if hit_length >= num_tokens:
|
||||
usable_length = self.coord.align_lookup_length(num_tokens - 1)
|
||||
if usable_length <= 0:
|
||||
return 0
|
||||
_masks, hit_length = self.coord.find_longest_cache_hit(
|
||||
block_hashes,
|
||||
usable_length,
|
||||
cached_block_pool,
|
||||
)
|
||||
return hit_length
|
||||
|
||||
def get_kv_events(self) -> list[BlockStored]:
|
||||
@@ -1605,11 +1621,11 @@ class LookupKeyServer:
|
||||
msg_type = bytes(all_frames[0])
|
||||
|
||||
if msg_type == LOOKUP_MSG:
|
||||
token_len = int.from_bytes(all_frames[1], byteorder="big")
|
||||
num_tokens = int.from_bytes(all_frames[1], byteorder="big")
|
||||
hash_len = int.from_bytes(all_frames[2], byteorder="big")
|
||||
blob = all_frames[3].buffer
|
||||
block_hashes = BlobBlockHashes(blob, hash_len)
|
||||
result = self.store_worker.lookup(token_len, block_hashes)
|
||||
result = self.store_worker.lookup(num_tokens, block_hashes)
|
||||
self.socket.send(result.to_bytes(4, "big"))
|
||||
|
||||
elif msg_type == RESET_MSG:
|
||||
@@ -1672,11 +1688,11 @@ class LookupKeyClient:
|
||||
)
|
||||
self.futures: dict[str, Future[int]] = {}
|
||||
|
||||
def _lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int:
|
||||
def _lookup(self, num_tokens: int, block_hashes: list[BlockHash]) -> int:
|
||||
hash_len = len(block_hashes[0]) if block_hashes else 0
|
||||
all_frames = (
|
||||
LOOKUP_MSG,
|
||||
token_len.to_bytes(4, byteorder="big"),
|
||||
num_tokens.to_bytes(4, byteorder="big"),
|
||||
hash_len.to_bytes(2, byteorder="big"),
|
||||
b"".join(block_hashes),
|
||||
)
|
||||
@@ -1687,7 +1703,7 @@ class LookupKeyClient:
|
||||
def lookup(
|
||||
self,
|
||||
req_id: str,
|
||||
token_len: int,
|
||||
num_tokens: int,
|
||||
block_hashes: list[BlockHash],
|
||||
non_block: bool = False,
|
||||
) -> int | None:
|
||||
@@ -1695,7 +1711,7 @@ class LookupKeyClient:
|
||||
so the caller retries on a later step."""
|
||||
future = self.futures.get(req_id)
|
||||
if future is None:
|
||||
future = self.executor.submit(self._lookup, token_len, list(block_hashes))
|
||||
future = self.executor.submit(self._lookup, num_tokens, list(block_hashes))
|
||||
self.futures[req_id] = future
|
||||
if non_block and not future.done():
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user