[KV Connector][Mooncake] Async lookup to reduce scheduler overhead (#45659)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -176,7 +176,7 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self,
|
||||
request: Request,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int, bool]:
|
||||
) -> tuple[int | None, bool]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
|
||||
@@ -54,9 +54,9 @@ class MooncakeStoreScheduler:
|
||||
):
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
self.kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get(
|
||||
"load_async", True
|
||||
)
|
||||
kvc_extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
|
||||
self.load_async = kvc_extra_config.get("load_async", True)
|
||||
self.lookup_async = kvc_extra_config.get("lookup_async", False)
|
||||
self.client = LookupKeyClient(vllm_config)
|
||||
|
||||
# Align with the engine's own scheduler_block_size and hash_block_size.
|
||||
@@ -75,14 +75,26 @@ class MooncakeStoreScheduler:
|
||||
self,
|
||||
request: Request,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int, bool]:
|
||||
"""Check for external KV cache hit."""
|
||||
) -> tuple[int | None, bool]:
|
||||
"""Check for external KV cache hit.
|
||||
|
||||
Returns ``(None, False)`` when an async lookup is still in flight,
|
||||
signaling the scheduler to retry this request on a later step.
|
||||
"""
|
||||
# Look up against the full prefill range, not just the prompt.
|
||||
token_len = request.num_tokens // self._block_size * self._block_size
|
||||
if token_len < self._block_size:
|
||||
return 0, False
|
||||
|
||||
num_external_hit_tokens = self.client.lookup(token_len, request.block_hashes)
|
||||
num_external_hit_tokens = self.client.lookup(
|
||||
request.request_id,
|
||||
token_len,
|
||||
request.block_hashes,
|
||||
non_block=self.lookup_async,
|
||||
)
|
||||
if num_external_hit_tokens is None:
|
||||
# Lookup not ready yet; scheduler will retry on a later step.
|
||||
return None, False
|
||||
|
||||
if num_external_hit_tokens == request.num_tokens:
|
||||
# Leave a sub-block tail uncomputed for sampling, on a block
|
||||
@@ -158,6 +170,7 @@ class MooncakeStoreScheduler:
|
||||
force_skip_save = self.kv_role == "kv_consumer"
|
||||
|
||||
for finished_req_id in scheduler_output.finished_req_ids:
|
||||
self.client.discard(finished_req_id)
|
||||
self.load_specs.pop(finished_req_id, None)
|
||||
self._request_trackers.pop(finished_req_id, None)
|
||||
self._unfinished_requests.pop(finished_req_id, None)
|
||||
|
||||
@@ -19,6 +19,7 @@ import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal, TypeVar
|
||||
|
||||
@@ -1560,7 +1561,13 @@ class LookupKeyClient:
|
||||
bind=False,
|
||||
)
|
||||
|
||||
def lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int:
|
||||
# Async lookup support
|
||||
self.executor = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="MooncakeLookupClient"
|
||||
)
|
||||
self.futures: dict[str, Future[int]] = {}
|
||||
|
||||
def _lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int:
|
||||
hash_strs = [h.hex() for h in block_hashes]
|
||||
hash_frames = self.encoder.encode(hash_strs)
|
||||
token_len_bytes = token_len.to_bytes(4, byteorder="big")
|
||||
@@ -1570,7 +1577,36 @@ class LookupKeyClient:
|
||||
result = int.from_bytes(resp, "big")
|
||||
return result
|
||||
|
||||
def reset(self) -> bool:
|
||||
def lookup(
|
||||
self,
|
||||
req_id: str,
|
||||
token_len: int,
|
||||
block_hashes: list[BlockHash],
|
||||
non_block: bool = False,
|
||||
) -> int | None:
|
||||
"""If non_block is True, will return None until the result is ready,
|
||||
so the caller retries on a later step."""
|
||||
future = self.futures.get(req_id)
|
||||
if future is None:
|
||||
future = self.executor.submit(self._lookup, token_len, list(block_hashes))
|
||||
self.futures[req_id] = future
|
||||
if non_block and not future.done():
|
||||
return None
|
||||
try:
|
||||
return future.result()
|
||||
except Exception as e:
|
||||
logger.error("Async Mooncake lookup failed for %s: %s", req_id, e)
|
||||
return 0
|
||||
finally:
|
||||
del self.futures[req_id]
|
||||
|
||||
def discard(self, req_id: str) -> None:
|
||||
"""Drop any cached/in-flight lookup for ``req_id`` (e.g. on abort)."""
|
||||
future = self.futures.pop(req_id, None)
|
||||
if future is not None:
|
||||
future.cancel()
|
||||
|
||||
def _reset(self) -> bool:
|
||||
"""Trigger ``store.remove_all(force=True)`` on worker rank 0.
|
||||
|
||||
Ordering assumption: caller MUST ensure no in-flight Mooncake
|
||||
@@ -1582,7 +1618,11 @@ class LookupKeyClient:
|
||||
resp = self.socket.recv()
|
||||
return bytes(resp) == RESP_OK
|
||||
|
||||
def reset(self) -> bool:
|
||||
return self.executor.submit(self._reset).result()
|
||||
|
||||
def close(self):
|
||||
self.executor.shutdown(wait=False, cancel_futures=True)
|
||||
self.socket.close(linger=0)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user