[MooncakeStore] Re-derive full external hits on stored boundaries (#49481)
Signed-off-by: Dao Le <Dao007forever@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Yifan Qiao
Claude
parent
a4904ba903
commit
a76df87db8
@@ -93,6 +93,9 @@ class MooncakeStoreCoordinator:
|
||||
self.eagle_group_ids = set(range(len(kv_cache_groups)))
|
||||
self._verify_and_split_kv_cache_groups()
|
||||
|
||||
def align_lookup_length(self, length: int) -> int:
|
||||
return length // self.lcm_block_size * self.lcm_block_size
|
||||
|
||||
def _verify_and_split_kv_cache_groups(self) -> None:
|
||||
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
|
||||
dispatches via spec_manager_map (we don't allocate managers).
|
||||
|
||||
@@ -10,7 +10,8 @@ Wire format (REQ/REP over IPC):
|
||||
Request: [msg_type: bytes] [payload_frames...]
|
||||
|
||||
msg_type == LOOKUP_MSG:
|
||||
frame 1: token_len (u32 big-endian, 4 bytes)
|
||||
frame 1: num_tokens (u32 big-endian, 4 bytes); the worker derives
|
||||
the aligned lookup length
|
||||
frame 2: hash_len (u16 big-endian, 2 bytes) — byte length of each
|
||||
fixed-size block hash (0 when there are no hashes)
|
||||
frame 3: raw block hashes concatenated back-to-back (each hash_len
|
||||
|
||||
@@ -80,14 +80,12 @@ class MooncakeStoreScheduler:
|
||||
Returns ``(None, False)`` when an async lookup is still in flight,
|
||||
signaling the scheduler to retry this request on a later step.
|
||||
"""
|
||||
# Look up against the full prefill range, not just the prompt.
|
||||
token_len = request.num_tokens // self._block_size * self._block_size
|
||||
if token_len < self._block_size:
|
||||
if request.num_tokens < self._block_size:
|
||||
return 0, False
|
||||
|
||||
num_external_hit_tokens = self.client.lookup(
|
||||
request.request_id,
|
||||
token_len,
|
||||
request.num_tokens,
|
||||
request.block_hashes,
|
||||
non_block=self.lookup_async,
|
||||
)
|
||||
@@ -95,14 +93,6 @@ class MooncakeStoreScheduler:
|
||||
# Lookup not ready yet; scheduler will retry on a later step.
|
||||
return None, False
|
||||
|
||||
if num_external_hit_tokens == request.num_tokens:
|
||||
# Leave a sub-block tail uncomputed for sampling, on a block
|
||||
# boundary so the recv-side load mask covers every yielded chunk.
|
||||
num_external_hit_tokens = max(
|
||||
0,
|
||||
(request.num_tokens - 1) // self._block_size * self._block_size,
|
||||
)
|
||||
|
||||
if num_external_hit_tokens < num_computed_tokens:
|
||||
need_to_allocate = 0
|
||||
else:
|
||||
|
||||
@@ -1467,11 +1467,14 @@ class MooncakeStoreWorker:
|
||||
|
||||
return finished_sending
|
||||
|
||||
def lookup(self, token_len: int, block_hashes: Sequence[BlockHash]) -> int:
|
||||
def lookup(self, num_tokens: int, block_hashes: Sequence[BlockHash]) -> int:
|
||||
"""Check how many prefix tokens exist in the store.
|
||||
|
||||
Checks across all rank-specific key namespaces that may be loaded.
|
||||
Checks across all rank-specific key namespaces that may be loaded. A
|
||||
hit covering all ``num_tokens`` is re-derived below the request end so
|
||||
the last token is recomputed for sampling.
|
||||
"""
|
||||
token_len = self.coord.align_lookup_length(num_tokens)
|
||||
if not block_hashes or token_len <= 0:
|
||||
return 0
|
||||
|
||||
@@ -1535,11 +1538,24 @@ class MooncakeStoreWorker:
|
||||
)
|
||||
}
|
||||
|
||||
cached_block_pool = ExternalCachedBlockPool(
|
||||
self.hash_block_size,
|
||||
exists_set,
|
||||
)
|
||||
_masks, hit_length = self.coord.find_longest_cache_hit(
|
||||
block_hashes,
|
||||
token_len,
|
||||
ExternalCachedBlockPool(self.hash_block_size, exists_set),
|
||||
cached_block_pool,
|
||||
)
|
||||
if hit_length >= num_tokens:
|
||||
usable_length = self.coord.align_lookup_length(num_tokens - 1)
|
||||
if usable_length <= 0:
|
||||
return 0
|
||||
_masks, hit_length = self.coord.find_longest_cache_hit(
|
||||
block_hashes,
|
||||
usable_length,
|
||||
cached_block_pool,
|
||||
)
|
||||
return hit_length
|
||||
|
||||
def get_kv_events(self) -> list[BlockStored]:
|
||||
@@ -1605,11 +1621,11 @@ class LookupKeyServer:
|
||||
msg_type = bytes(all_frames[0])
|
||||
|
||||
if msg_type == LOOKUP_MSG:
|
||||
token_len = int.from_bytes(all_frames[1], byteorder="big")
|
||||
num_tokens = int.from_bytes(all_frames[1], byteorder="big")
|
||||
hash_len = int.from_bytes(all_frames[2], byteorder="big")
|
||||
blob = all_frames[3].buffer
|
||||
block_hashes = BlobBlockHashes(blob, hash_len)
|
||||
result = self.store_worker.lookup(token_len, block_hashes)
|
||||
result = self.store_worker.lookup(num_tokens, block_hashes)
|
||||
self.socket.send(result.to_bytes(4, "big"))
|
||||
|
||||
elif msg_type == RESET_MSG:
|
||||
@@ -1672,11 +1688,11 @@ class LookupKeyClient:
|
||||
)
|
||||
self.futures: dict[str, Future[int]] = {}
|
||||
|
||||
def _lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int:
|
||||
def _lookup(self, num_tokens: int, block_hashes: list[BlockHash]) -> int:
|
||||
hash_len = len(block_hashes[0]) if block_hashes else 0
|
||||
all_frames = (
|
||||
LOOKUP_MSG,
|
||||
token_len.to_bytes(4, byteorder="big"),
|
||||
num_tokens.to_bytes(4, byteorder="big"),
|
||||
hash_len.to_bytes(2, byteorder="big"),
|
||||
b"".join(block_hashes),
|
||||
)
|
||||
@@ -1687,7 +1703,7 @@ class LookupKeyClient:
|
||||
def lookup(
|
||||
self,
|
||||
req_id: str,
|
||||
token_len: int,
|
||||
num_tokens: int,
|
||||
block_hashes: list[BlockHash],
|
||||
non_block: bool = False,
|
||||
) -> int | None:
|
||||
@@ -1695,7 +1711,7 @@ class LookupKeyClient:
|
||||
so the caller retries on a later step."""
|
||||
future = self.futures.get(req_id)
|
||||
if future is None:
|
||||
future = self.executor.submit(self._lookup, token_len, list(block_hashes))
|
||||
future = self.executor.submit(self._lookup, num_tokens, list(block_hashes))
|
||||
self.futures[req_id] = future
|
||||
if non_block and not future.done():
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user