[Perf][KVConnector][Mooncake] Parallelize KV load with a receive-thread pool (#45971)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
dda3aca47f
commit
9e88e969c0
@@ -4,6 +4,7 @@
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import queue
|
||||
import sys
|
||||
import threading
|
||||
import types
|
||||
@@ -760,7 +761,7 @@ def test_store_worker_get_block_ids_with_load_errors_delegates_to_recv_thread():
|
||||
recv_thread = MagicMock()
|
||||
recv_thread.get_and_clear_block_ids_with_load_errors.return_value = {3, 4}
|
||||
w = _make_bare_worker()
|
||||
w.kv_recv_thread = recv_thread
|
||||
w.kv_recv_threads = [recv_thread]
|
||||
|
||||
assert w.get_block_ids_with_load_errors() == {3, 4}
|
||||
recv_thread.get_and_clear_block_ids_with_load_errors.assert_called_once_with()
|
||||
@@ -1574,7 +1575,9 @@ def _make_bare_worker(
|
||||
worker.put_step = 1
|
||||
worker.enable_kv_events = False
|
||||
worker.kv_send_thread = None
|
||||
worker.kv_recv_thread = None
|
||||
worker.kv_recv_threads = []
|
||||
worker.num_recv_threads = 1
|
||||
worker.recv_request_queue = queue.Queue()
|
||||
worker.tp_size = 1
|
||||
worker.num_kv_head = 1
|
||||
worker.pp_size = 1
|
||||
|
||||
@@ -359,6 +359,7 @@ class KVTransferThread(threading.Thread):
|
||||
ready_event: threading.Event,
|
||||
name: str,
|
||||
record_operation: Callable[..., None] | None = None,
|
||||
request_queue: queue.Queue[Any] | None = None,
|
||||
):
|
||||
super().__init__(daemon=True, name=name)
|
||||
self.store = store
|
||||
@@ -368,7 +369,7 @@ class KVTransferThread(threading.Thread):
|
||||
self.token_databases = token_databases
|
||||
self._record_operation_cb = record_operation
|
||||
self.done_task_lock = threading.Lock()
|
||||
self.request_queue: queue.Queue[Any] = queue.Queue()
|
||||
self.request_queue: queue.Queue[Any] = request_queue or queue.Queue()
|
||||
self.finished_requests: set[str] = set()
|
||||
self.kv_event_lock = threading.Lock()
|
||||
self.kv_events: list[BlockStored] = []
|
||||
@@ -749,6 +750,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
ready_event: threading.Event,
|
||||
disk_offload_buffer_budget_bytes: int | None = None,
|
||||
record_operation: Callable[..., None] | None = None,
|
||||
request_queue: queue.Queue[Any] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
store,
|
||||
@@ -758,6 +760,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
ready_event,
|
||||
name="KVCacheStoreRecvingThread",
|
||||
record_operation=record_operation,
|
||||
request_queue=request_queue,
|
||||
)
|
||||
# _invalid_block_ids can be access by both the Worker and RecvingThread
|
||||
self._invalid_block_ids_lock = threading.Lock()
|
||||
@@ -1101,7 +1104,10 @@ class MooncakeStoreWorker:
|
||||
self.enable_kv_events = True
|
||||
|
||||
self.kv_send_thread: KVCacheStoreSendingThread | None = None
|
||||
self.kv_recv_thread: KVCacheStoreRecvingThread | None = None
|
||||
# Pool of load-receive threads
|
||||
self.kv_recv_threads: list[KVCacheStoreRecvingThread] = []
|
||||
self.num_recv_threads = max(1, envs.VLLM_MOONCAKE_LOAD_RECV_THREADS)
|
||||
self.recv_request_queue: queue.Queue[ReqMeta] = queue.Queue()
|
||||
self.finished_store_req: set[str] = set()
|
||||
self._kv_connector_stats_lock = threading.Lock()
|
||||
self.kv_connector_stats = MooncakeStoreConnectorStats()
|
||||
@@ -1260,19 +1266,30 @@ class MooncakeStoreWorker:
|
||||
)
|
||||
self.kv_send_thread.start()
|
||||
|
||||
ready_event_recving = threading.Event()
|
||||
self.kv_recv_thread = KVCacheStoreRecvingThread(
|
||||
self.store,
|
||||
self.coord,
|
||||
self.token_dbs,
|
||||
self.block_size,
|
||||
self.tp_rank,
|
||||
ready_event_recving,
|
||||
disk_offload_buffer_budget_bytes=self.disk_offload_buffer_budget_bytes,
|
||||
record_operation=self._record_kv_connector_operation,
|
||||
self.kv_recv_threads = []
|
||||
ready_events_recving = []
|
||||
for i in range(self.num_recv_threads):
|
||||
ready_event_recving = threading.Event()
|
||||
recv_thread = KVCacheStoreRecvingThread(
|
||||
self.store,
|
||||
self.coord,
|
||||
self.token_dbs,
|
||||
self.block_size,
|
||||
self.tp_rank,
|
||||
ready_event_recving,
|
||||
disk_offload_buffer_budget_bytes=self.disk_offload_buffer_budget_bytes,
|
||||
record_operation=self._record_kv_connector_operation,
|
||||
request_queue=self.recv_request_queue,
|
||||
)
|
||||
recv_thread.name = f"KVCacheStoreRecvingThread-{i}"
|
||||
recv_thread.start()
|
||||
self.kv_recv_threads.append(recv_thread)
|
||||
ready_events_recving.append(ready_event_recving)
|
||||
for ready_event_recving in ready_events_recving:
|
||||
ready_event_recving.wait()
|
||||
logger.info(
|
||||
"Started %d Mooncake KV-load receive thread(s)", self.num_recv_threads
|
||||
)
|
||||
self.kv_recv_thread.start()
|
||||
ready_event_recving.wait()
|
||||
|
||||
def start_load_kv(
|
||||
self,
|
||||
@@ -1306,9 +1323,7 @@ class MooncakeStoreWorker:
|
||||
continue
|
||||
|
||||
load_spec.token_len = load_spec.kvpool_cached_tokens
|
||||
|
||||
assert self.kv_recv_thread is not None
|
||||
self.kv_recv_thread.add_request(request)
|
||||
self.recv_request_queue.put(request)
|
||||
|
||||
assert self.load_async, "load_async must be True for better performance."
|
||||
# Issue stores with CUDA event synchronization
|
||||
@@ -1335,11 +1350,10 @@ class MooncakeStoreWorker:
|
||||
else set()
|
||||
)
|
||||
|
||||
done_recving = (
|
||||
self.kv_recv_thread.get_and_clear_finished_requests()
|
||||
if self.load_async and self.kv_recv_thread is not None
|
||||
else set()
|
||||
)
|
||||
done_recving: set[str] = set()
|
||||
if self.load_async:
|
||||
for recv_thread in self.kv_recv_threads:
|
||||
done_recving |= recv_thread.get_and_clear_finished_requests()
|
||||
|
||||
logger.debug(
|
||||
"Completed send: %d, recv: %d, tp_rank: %d",
|
||||
@@ -1350,9 +1364,10 @@ class MooncakeStoreWorker:
|
||||
return done_sending, done_recving
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
if self.kv_recv_thread is None:
|
||||
return set()
|
||||
return self.kv_recv_thread.get_and_clear_block_ids_with_load_errors()
|
||||
block_ids: set[int] = set()
|
||||
for recv_thread in self.kv_recv_threads:
|
||||
block_ids |= recv_thread.get_and_clear_block_ids_with_load_errors()
|
||||
return block_ids
|
||||
|
||||
def _record_kv_connector_operation(
|
||||
self,
|
||||
|
||||
@@ -200,6 +200,7 @@ if TYPE_CHECKING:
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998
|
||||
VLLM_MOONCAKE_STORE_TIER_LOG: bool = False
|
||||
VLLM_MOONCAKE_LOAD_RECV_THREADS: int = 1
|
||||
VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO: float = 0.9
|
||||
MOONCAKE_PREFERRED_SEGMENT: str | None = None
|
||||
MOONCAKE_REQUESTER_LOCAL_HOSTNAME: str | None = None
|
||||
@@ -1531,6 +1532,14 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_MOONCAKE_STORE_TIER_LOG": lambda: (
|
||||
os.getenv("VLLM_MOONCAKE_STORE_TIER_LOG", "False").lower() in ("true", "1")
|
||||
),
|
||||
# Number of parallel KV-load receive threads per worker rank. Lets the
|
||||
# per-request control overhead (Python prep + master key lookup) of one
|
||||
# request overlap with the RDMA transfer of another, keeping the transfer
|
||||
# engine's queue pairs busy. Helps when that overhead is significant or
|
||||
# per-request batches are too small to saturate the link on their own.
|
||||
"VLLM_MOONCAKE_LOAD_RECV_THREADS": lambda: int(
|
||||
os.getenv("VLLM_MOONCAKE_LOAD_RECV_THREADS", "1")
|
||||
),
|
||||
# Fraction of the owner's DirectIO staging buffer to fill per GET batch.
|
||||
"VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO": lambda: float(
|
||||
os.getenv("VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO", "0.9")
|
||||
|
||||
Reference in New Issue
Block a user