[MooncakeStore] Re-derive full external hits on stored boundaries (#49481)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Dao007forever
2026-07-23 18:12:28 +08:00
committed by GitHub
co-authored by Yifan Qiao Claude
parent a4904ba903
commit a76df87db8
8 changed files with 110 additions and 38 deletions
@@ -93,6 +93,9 @@ class MooncakeStoreCoordinator:
self.eagle_group_ids = set(range(len(kv_cache_groups)))
self._verify_and_split_kv_cache_groups()
def align_lookup_length(self, length: int) -> int:
return length // self.lcm_block_size * self.lcm_block_size
def _verify_and_split_kv_cache_groups(self) -> None:
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
dispatches via spec_manager_map (we don't allocate managers).
@@ -10,7 +10,8 @@ Wire format (REQ/REP over IPC):
Request: [msg_type: bytes] [payload_frames...]
msg_type == LOOKUP_MSG:
frame 1: token_len (u32 big-endian, 4 bytes)
frame 1: num_tokens (u32 big-endian, 4 bytes); the worker derives
the aligned lookup length
frame 2: hash_len (u16 big-endian, 2 bytes) — byte length of each
fixed-size block hash (0 when there are no hashes)
frame 3: raw block hashes concatenated back-to-back (each hash_len
@@ -80,14 +80,12 @@ class MooncakeStoreScheduler:
Returns ``(None, False)`` when an async lookup is still in flight,
signaling the scheduler to retry this request on a later step.
"""
# Look up against the full prefill range, not just the prompt.
token_len = request.num_tokens // self._block_size * self._block_size
if token_len < self._block_size:
if request.num_tokens < self._block_size:
return 0, False
num_external_hit_tokens = self.client.lookup(
request.request_id,
token_len,
request.num_tokens,
request.block_hashes,
non_block=self.lookup_async,
)
@@ -95,14 +93,6 @@ class MooncakeStoreScheduler:
# Lookup not ready yet; scheduler will retry on a later step.
return None, False
if num_external_hit_tokens == request.num_tokens:
# Leave a sub-block tail uncomputed for sampling, on a block
# boundary so the recv-side load mask covers every yielded chunk.
num_external_hit_tokens = max(
0,
(request.num_tokens - 1) // self._block_size * self._block_size,
)
if num_external_hit_tokens < num_computed_tokens:
need_to_allocate = 0
else:
@@ -1467,11 +1467,14 @@ class MooncakeStoreWorker:
return finished_sending
def lookup(self, token_len: int, block_hashes: Sequence[BlockHash]) -> int:
def lookup(self, num_tokens: int, block_hashes: Sequence[BlockHash]) -> int:
"""Check how many prefix tokens exist in the store.
Checks across all rank-specific key namespaces that may be loaded.
Checks across all rank-specific key namespaces that may be loaded. A
hit covering all ``num_tokens`` is re-derived below the request end so
the last token is recomputed for sampling.
"""
token_len = self.coord.align_lookup_length(num_tokens)
if not block_hashes or token_len <= 0:
return 0
@@ -1535,11 +1538,24 @@ class MooncakeStoreWorker:
)
}
cached_block_pool = ExternalCachedBlockPool(
self.hash_block_size,
exists_set,
)
_masks, hit_length = self.coord.find_longest_cache_hit(
block_hashes,
token_len,
ExternalCachedBlockPool(self.hash_block_size, exists_set),
cached_block_pool,
)
if hit_length >= num_tokens:
usable_length = self.coord.align_lookup_length(num_tokens - 1)
if usable_length <= 0:
return 0
_masks, hit_length = self.coord.find_longest_cache_hit(
block_hashes,
usable_length,
cached_block_pool,
)
return hit_length
def get_kv_events(self) -> list[BlockStored]:
@@ -1605,11 +1621,11 @@ class LookupKeyServer:
msg_type = bytes(all_frames[0])
if msg_type == LOOKUP_MSG:
token_len = int.from_bytes(all_frames[1], byteorder="big")
num_tokens = int.from_bytes(all_frames[1], byteorder="big")
hash_len = int.from_bytes(all_frames[2], byteorder="big")
blob = all_frames[3].buffer
block_hashes = BlobBlockHashes(blob, hash_len)
result = self.store_worker.lookup(token_len, block_hashes)
result = self.store_worker.lookup(num_tokens, block_hashes)
self.socket.send(result.to_bytes(4, "big"))
elif msg_type == RESET_MSG:
@@ -1672,11 +1688,11 @@ class LookupKeyClient:
)
self.futures: dict[str, Future[int]] = {}
def _lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int:
def _lookup(self, num_tokens: int, block_hashes: list[BlockHash]) -> int:
hash_len = len(block_hashes[0]) if block_hashes else 0
all_frames = (
LOOKUP_MSG,
token_len.to_bytes(4, byteorder="big"),
num_tokens.to_bytes(4, byteorder="big"),
hash_len.to_bytes(2, byteorder="big"),
b"".join(block_hashes),
)
@@ -1687,7 +1703,7 @@ class LookupKeyClient:
def lookup(
self,
req_id: str,
token_len: int,
num_tokens: int,
block_hashes: list[BlockHash],
non_block: bool = False,
) -> int | None:
@@ -1695,7 +1711,7 @@ class LookupKeyClient:
so the caller retries on a later step."""
future = self.futures.get(req_id)
if future is None:
future = self.executor.submit(self._lookup, token_len, list(block_hashes))
future = self.executor.submit(self._lookup, num_tokens, list(block_hashes))
self.futures[req_id] = future
if non_block and not future.done():
return None