[KV Connector][Mooncake] Async lookup to reduce scheduler overhead (#45659)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import threading
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
@@ -406,7 +408,9 @@ def test_lookup_key_client_lookup_prepends_typed_tag():
|
||||
fake_socket = mock_make_socket.return_value
|
||||
fake_socket.recv.return_value = (5).to_bytes(4, "big")
|
||||
|
||||
assert client.lookup(token_len=128, block_hashes=[]) == 5
|
||||
# Blocking lookup (non_block defaults to False) runs on the executor and
|
||||
# returns the resolved hit length.
|
||||
assert client.lookup("req0", token_len=128, block_hashes=[]) == 5
|
||||
|
||||
sent_frames = fake_socket.send_multipart.call_args[0][0]
|
||||
assert sent_frames[0] == protocol.LOOKUP_MSG
|
||||
@@ -435,6 +439,127 @@ def test_lookup_key_client_reset_uses_typed_protocol():
|
||||
assert client.reset() is False
|
||||
|
||||
|
||||
def _poll_lookup(client, req_id, token_len=128, block_hashes=(), timeout=5.0):
|
||||
"""Drive non-blocking lookup until the executor completes it."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = client.lookup(req_id, token_len, list(block_hashes), non_block=True)
|
||||
if result is not None:
|
||||
return result
|
||||
time.sleep(0.005)
|
||||
return None
|
||||
|
||||
|
||||
def _gated_recv(gate: threading.Event, value: int):
|
||||
"""Mock recv side-effect that blocks until ``gate`` is set, so the
|
||||
executor's lookup can be held pending deterministically."""
|
||||
|
||||
def recv():
|
||||
gate.wait()
|
||||
return value.to_bytes(4, "big")
|
||||
|
||||
return recv
|
||||
|
||||
|
||||
def test_lookup_key_client_non_block_lookup_async():
|
||||
"""Non-blocking lookup defers to the executor: None first, hit once the
|
||||
Future resolves."""
|
||||
vllm_config = _make_vllm_config()
|
||||
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"worker.make_zmq_socket"
|
||||
) as mock_make_socket:
|
||||
client = worker.LookupKeyClient(vllm_config)
|
||||
|
||||
fake_socket = mock_make_socket.return_value
|
||||
# Hold the executor's lookup pending until we release the gate.
|
||||
gate = threading.Event()
|
||||
fake_socket.recv.side_effect = _gated_recv(gate, 7)
|
||||
|
||||
# First query submits the lookup and returns None while it is in flight.
|
||||
assert client.lookup("req1", 128, [], non_block=True) is None
|
||||
# Release the executor; a later poll returns the hit length.
|
||||
gate.set()
|
||||
assert _poll_lookup(client, "req1") == 7
|
||||
# Future is consumed (popped) on read.
|
||||
assert "req1" not in client.futures
|
||||
|
||||
|
||||
def test_lookup_key_client_discard_clears_state():
|
||||
"""discard() drops a completed lookup Future so it is not served stale."""
|
||||
vllm_config = _make_vllm_config()
|
||||
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"worker.make_zmq_socket"
|
||||
) as mock_make_socket:
|
||||
client = worker.LookupKeyClient(vllm_config)
|
||||
|
||||
fake_socket = mock_make_socket.return_value
|
||||
gate = threading.Event()
|
||||
fake_socket.recv.side_effect = _gated_recv(gate, 9)
|
||||
|
||||
# Submit while gated so the call returns None and the Future stays in
|
||||
# `futures` (unconsumed) once it resolves.
|
||||
assert client.lookup("req2", 128, [], non_block=True) is None
|
||||
gate.set()
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline:
|
||||
if client.futures["req2"].done():
|
||||
break
|
||||
time.sleep(0.005)
|
||||
# discard() drops the completed result before any lookup consumes it.
|
||||
client.discard("req2")
|
||||
assert "req2" not in client.futures
|
||||
# A fresh query re-submits rather than returning a stale value: hold the
|
||||
# gate so the resubmitted lookup stays in flight.
|
||||
gate.clear()
|
||||
assert client.lookup("req2", 128, [], non_block=True) is None
|
||||
gate.set() # release the executor so the worker thread can drain
|
||||
|
||||
|
||||
def test_get_num_new_matched_tokens_async_defers_then_reports():
|
||||
"""Async lookup returns (None, False) until ready, then the hit count."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeStoreConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"lookup_async": True},
|
||||
)
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"scheduler.LookupKeyClient"
|
||||
) as mock_client_cls,
|
||||
):
|
||||
sched = scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
|
||||
|
||||
assert sched.lookup_async is True
|
||||
mock_client = mock_client_cls.return_value
|
||||
|
||||
block_size = sched._block_size
|
||||
request = MagicMock()
|
||||
request.request_id = "r1"
|
||||
request.num_tokens = 4 * block_size
|
||||
request.block_hashes = []
|
||||
|
||||
# Lookup not ready -> defer.
|
||||
mock_client.lookup.return_value = None
|
||||
assert sched.get_num_new_matched_tokens(request, 0) == (None, False)
|
||||
assert "r1" not in sched.load_specs
|
||||
|
||||
# Lookup ready with a hit -> report need_to_allocate + async-load flag.
|
||||
hit = 3 * block_size
|
||||
mock_client.lookup.return_value = hit
|
||||
need, load_async = sched.get_num_new_matched_tokens(request, 0)
|
||||
assert need == hit
|
||||
assert load_async == sched.load_async
|
||||
assert sched.load_specs["r1"].kvpool_cached_tokens == hit
|
||||
|
||||
|
||||
def test_protocol_tags_are_distinct_and_non_empty():
|
||||
"""Protocol tags must be unique and non-empty to avoid collision."""
|
||||
tags = {protocol.LOOKUP_MSG, protocol.RESET_MSG}
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler impor
|
||||
def _make_bare_scheduler() -> MooncakeStoreScheduler:
|
||||
scheduler = object.__new__(MooncakeStoreScheduler)
|
||||
scheduler.kv_role = "kv_both"
|
||||
scheduler.lookup_async = False
|
||||
scheduler._block_size = 16
|
||||
scheduler.load_specs = {}
|
||||
scheduler._preempted_req_ids = set()
|
||||
@@ -405,7 +406,13 @@ class _StubLookupClient:
|
||||
def __init__(self, hit_tokens: int) -> None:
|
||||
self._hit_tokens = hit_tokens
|
||||
|
||||
def lookup(self, token_len: int, block_hashes: list[bytes]) -> int:
|
||||
def lookup(
|
||||
self,
|
||||
req_id: str,
|
||||
token_len: int,
|
||||
block_hashes: list[bytes],
|
||||
non_block: bool = False,
|
||||
) -> int:
|
||||
return self._hit_tokens
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user