[Perf][KVConnector][Mooncake] Parallelize KV load with a receive-thread pool (#45971)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Yifan Qiao
2026-06-24 18:25:12 -07:00
committed by GitHub
co-authored by Claude mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent dda3aca47f
commit 9e88e969c0
3 changed files with 54 additions and 27 deletions
@@ -4,6 +4,7 @@
import json
import logging
import math
import queue
import sys
import threading
import types
@@ -760,7 +761,7 @@ def test_store_worker_get_block_ids_with_load_errors_delegates_to_recv_thread():
recv_thread = MagicMock()
recv_thread.get_and_clear_block_ids_with_load_errors.return_value = {3, 4}
w = _make_bare_worker()
w.kv_recv_thread = recv_thread
w.kv_recv_threads = [recv_thread]
assert w.get_block_ids_with_load_errors() == {3, 4}
recv_thread.get_and_clear_block_ids_with_load_errors.assert_called_once_with()
@@ -1574,7 +1575,9 @@ def _make_bare_worker(
worker.put_step = 1
worker.enable_kv_events = False
worker.kv_send_thread = None
worker.kv_recv_thread = None
worker.kv_recv_threads = []
worker.num_recv_threads = 1
worker.recv_request_queue = queue.Queue()
worker.tp_size = 1
worker.num_kv_head = 1
worker.pp_size = 1
@@ -359,6 +359,7 @@ class KVTransferThread(threading.Thread):
ready_event: threading.Event,
name: str,
record_operation: Callable[..., None] | None = None,
request_queue: queue.Queue[Any] | None = None,
):
super().__init__(daemon=True, name=name)
self.store = store
@@ -368,7 +369,7 @@ class KVTransferThread(threading.Thread):
self.token_databases = token_databases
self._record_operation_cb = record_operation
self.done_task_lock = threading.Lock()
self.request_queue: queue.Queue[Any] = queue.Queue()
self.request_queue: queue.Queue[Any] = request_queue or queue.Queue()
self.finished_requests: set[str] = set()
self.kv_event_lock = threading.Lock()
self.kv_events: list[BlockStored] = []
@@ -749,6 +750,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
ready_event: threading.Event,
disk_offload_buffer_budget_bytes: int | None = None,
record_operation: Callable[..., None] | None = None,
request_queue: queue.Queue[Any] | None = None,
):
super().__init__(
store,
@@ -758,6 +760,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
ready_event,
name="KVCacheStoreRecvingThread",
record_operation=record_operation,
request_queue=request_queue,
)
# _invalid_block_ids can be access by both the Worker and RecvingThread
self._invalid_block_ids_lock = threading.Lock()
@@ -1101,7 +1104,10 @@ class MooncakeStoreWorker:
self.enable_kv_events = True
self.kv_send_thread: KVCacheStoreSendingThread | None = None
self.kv_recv_thread: KVCacheStoreRecvingThread | None = None
# Pool of load-receive threads
self.kv_recv_threads: list[KVCacheStoreRecvingThread] = []
self.num_recv_threads = max(1, envs.VLLM_MOONCAKE_LOAD_RECV_THREADS)
self.recv_request_queue: queue.Queue[ReqMeta] = queue.Queue()
self.finished_store_req: set[str] = set()
self._kv_connector_stats_lock = threading.Lock()
self.kv_connector_stats = MooncakeStoreConnectorStats()
@@ -1260,19 +1266,30 @@ class MooncakeStoreWorker:
)
self.kv_send_thread.start()
ready_event_recving = threading.Event()
self.kv_recv_thread = KVCacheStoreRecvingThread(
self.store,
self.coord,
self.token_dbs,
self.block_size,
self.tp_rank,
ready_event_recving,
disk_offload_buffer_budget_bytes=self.disk_offload_buffer_budget_bytes,
record_operation=self._record_kv_connector_operation,
self.kv_recv_threads = []
ready_events_recving = []
for i in range(self.num_recv_threads):
ready_event_recving = threading.Event()
recv_thread = KVCacheStoreRecvingThread(
self.store,
self.coord,
self.token_dbs,
self.block_size,
self.tp_rank,
ready_event_recving,
disk_offload_buffer_budget_bytes=self.disk_offload_buffer_budget_bytes,
record_operation=self._record_kv_connector_operation,
request_queue=self.recv_request_queue,
)
recv_thread.name = f"KVCacheStoreRecvingThread-{i}"
recv_thread.start()
self.kv_recv_threads.append(recv_thread)
ready_events_recving.append(ready_event_recving)
for ready_event_recving in ready_events_recving:
ready_event_recving.wait()
logger.info(
"Started %d Mooncake KV-load receive thread(s)", self.num_recv_threads
)
self.kv_recv_thread.start()
ready_event_recving.wait()
def start_load_kv(
self,
@@ -1306,9 +1323,7 @@ class MooncakeStoreWorker:
continue
load_spec.token_len = load_spec.kvpool_cached_tokens
assert self.kv_recv_thread is not None
self.kv_recv_thread.add_request(request)
self.recv_request_queue.put(request)
assert self.load_async, "load_async must be True for better performance."
# Issue stores with CUDA event synchronization
@@ -1335,11 +1350,10 @@ class MooncakeStoreWorker:
else set()
)
done_recving = (
self.kv_recv_thread.get_and_clear_finished_requests()
if self.load_async and self.kv_recv_thread is not None
else set()
)
done_recving: set[str] = set()
if self.load_async:
for recv_thread in self.kv_recv_threads:
done_recving |= recv_thread.get_and_clear_finished_requests()
logger.debug(
"Completed send: %d, recv: %d, tp_rank: %d",
@@ -1350,9 +1364,10 @@ class MooncakeStoreWorker:
return done_sending, done_recving
def get_block_ids_with_load_errors(self) -> set[int]:
if self.kv_recv_thread is None:
return set()
return self.kv_recv_thread.get_and_clear_block_ids_with_load_errors()
block_ids: set[int] = set()
for recv_thread in self.kv_recv_threads:
block_ids |= recv_thread.get_and_clear_block_ids_with_load_errors()
return block_ids
def _record_kv_connector_operation(
self,
+9
View File
@@ -200,6 +200,7 @@ if TYPE_CHECKING:
VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600
VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998
VLLM_MOONCAKE_STORE_TIER_LOG: bool = False
VLLM_MOONCAKE_LOAD_RECV_THREADS: int = 1
VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO: float = 0.9
MOONCAKE_PREFERRED_SEGMENT: str | None = None
MOONCAKE_REQUESTER_LOCAL_HOSTNAME: str | None = None
@@ -1531,6 +1532,14 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_MOONCAKE_STORE_TIER_LOG": lambda: (
os.getenv("VLLM_MOONCAKE_STORE_TIER_LOG", "False").lower() in ("true", "1")
),
# Number of parallel KV-load receive threads per worker rank. Lets the
# per-request control overhead (Python prep + master key lookup) of one
# request overlap with the RDMA transfer of another, keeping the transfer
# engine's queue pairs busy. Helps when that overhead is significant or
# per-request batches are too small to saturate the link on their own.
"VLLM_MOONCAKE_LOAD_RECV_THREADS": lambda: int(
os.getenv("VLLM_MOONCAKE_LOAD_RECV_THREADS", "1")
),
# Fraction of the owner's DirectIO staging buffer to fill per GET batch.
"VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO": lambda: float(
os.getenv("VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO", "0.9")