[KV Connector][Mooncake] Async lookup to reduce scheduler overhead (#45659)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Yifan Qiao
2026-06-18 21:44:02 +00:00
committed by GitHub
co-authored by Nick Hill
parent 4ce2d01453
commit 35e4dd4a69
6 changed files with 197 additions and 11 deletions
@@ -176,7 +176,7 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
self,
request: Request,
num_computed_tokens: int,
) -> tuple[int, bool]:
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
@@ -54,9 +54,9 @@ class MooncakeStoreScheduler:
):
assert vllm_config.kv_transfer_config is not None
self.kv_role = vllm_config.kv_transfer_config.kv_role
self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get(
"load_async", True
)
kvc_extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
self.load_async = kvc_extra_config.get("load_async", True)
self.lookup_async = kvc_extra_config.get("lookup_async", False)
self.client = LookupKeyClient(vllm_config)
# Align with the engine's own scheduler_block_size and hash_block_size.
@@ -75,14 +75,26 @@ class MooncakeStoreScheduler:
self,
request: Request,
num_computed_tokens: int,
) -> tuple[int, bool]:
"""Check for external KV cache hit."""
) -> tuple[int | None, bool]:
"""Check for external KV cache hit.
Returns ``(None, False)`` when an async lookup is still in flight,
signaling the scheduler to retry this request on a later step.
"""
# Look up against the full prefill range, not just the prompt.
token_len = request.num_tokens // self._block_size * self._block_size
if token_len < self._block_size:
return 0, False
num_external_hit_tokens = self.client.lookup(token_len, request.block_hashes)
num_external_hit_tokens = self.client.lookup(
request.request_id,
token_len,
request.block_hashes,
non_block=self.lookup_async,
)
if num_external_hit_tokens is None:
# Lookup not ready yet; scheduler will retry on a later step.
return None, False
if num_external_hit_tokens == request.num_tokens:
# Leave a sub-block tail uncomputed for sampling, on a block
@@ -158,6 +170,7 @@ class MooncakeStoreScheduler:
force_skip_save = self.kv_role == "kv_consumer"
for finished_req_id in scheduler_output.finished_req_ids:
self.client.discard(finished_req_id)
self.load_specs.pop(finished_req_id, None)
self._request_trackers.pop(finished_req_id, None)
self._unfinished_requests.pop(finished_req_id, None)
@@ -19,6 +19,7 @@ import threading
import time
from collections import defaultdict
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
from typing import Any, Literal, TypeVar
@@ -1560,7 +1561,13 @@ class LookupKeyClient:
bind=False,
)
def lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int:
# Async lookup support
self.executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="MooncakeLookupClient"
)
self.futures: dict[str, Future[int]] = {}
def _lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int:
hash_strs = [h.hex() for h in block_hashes]
hash_frames = self.encoder.encode(hash_strs)
token_len_bytes = token_len.to_bytes(4, byteorder="big")
@@ -1570,7 +1577,36 @@ class LookupKeyClient:
result = int.from_bytes(resp, "big")
return result
def reset(self) -> bool:
def lookup(
self,
req_id: str,
token_len: int,
block_hashes: list[BlockHash],
non_block: bool = False,
) -> int | None:
"""If non_block is True, will return None until the result is ready,
so the caller retries on a later step."""
future = self.futures.get(req_id)
if future is None:
future = self.executor.submit(self._lookup, token_len, list(block_hashes))
self.futures[req_id] = future
if non_block and not future.done():
return None
try:
return future.result()
except Exception as e:
logger.error("Async Mooncake lookup failed for %s: %s", req_id, e)
return 0
finally:
del self.futures[req_id]
def discard(self, req_id: str) -> None:
"""Drop any cached/in-flight lookup for ``req_id`` (e.g. on abort)."""
future = self.futures.pop(req_id, None)
if future is not None:
future.cancel()
def _reset(self) -> bool:
"""Trigger ``store.remove_all(force=True)`` on worker rank 0.
Ordering assumption: caller MUST ensure no in-flight Mooncake
@@ -1582,7 +1618,11 @@ class LookupKeyClient:
resp = self.socket.recv()
return bytes(resp) == RESP_OK
def reset(self) -> bool:
return self.executor.submit(self._reset).result()
def close(self):
self.executor.shutdown(wait=False, cancel_futures=True)
self.socket.close(linger=0)