[KV Connector] Propagate MooncakeStore load failures (#42788)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
This commit is contained in:
Dao007forever
2026-05-25 22:12:15 -07:00
committed by GitHub
parent 7966fc7233
commit c2a4005c70
4 changed files with 143 additions and 15 deletions
@@ -270,6 +270,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
return self.connector_worker.get_finished(finished_req_ids, metadata)
def get_block_ids_with_load_errors(self) -> set[int]:
assert self.connector_worker is not None
return self.connector_worker.get_block_ids_with_load_errors()
def get_kv_connector_kv_cache_events(
self,
) -> MooncakeStoreKVEvents | None:
@@ -20,7 +20,7 @@ import time
from collections import defaultdict
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any, Literal
from typing import Any, Literal, TypeVar
import regex as re
import torch
@@ -68,6 +68,12 @@ DEFAULT_GLOBAL_SEGMENT_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
DEFAULT_LOCAL_BUFFER_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
MOONCAKE_NO_AVAILABLE_HANDLE = -200
_T = TypeVar("_T")
def _rotate_list(values: list[_T], offset: int) -> list[_T]:
return values[offset:] + values[:offset]
# Mirrors FileStorageConfig::local_buffer_size in Mooncake C++.
DEFAULT_MOONCAKE_DISK_STAGING_BUFFER_BYTES = 1280 * 1024 * 1024
@@ -711,6 +717,9 @@ class KVCacheStoreRecvingThread(KVTransferThread):
name="KVCacheStoreRecvingThread",
record_operation=record_operation,
)
# _invalid_block_ids can be access by both the Worker and RecvingThread
self._invalid_block_ids_lock = threading.Lock()
self._invalid_block_ids: set[int] = set()
self.disk_offload_buffer_budget_bytes = disk_offload_buffer_budget_bytes
self.usable_disk_offload_buffer_budget_bytes = (
None
@@ -721,6 +730,16 @@ class KVCacheStoreRecvingThread(KVTransferThread):
)
self.coord = coord
def _add_load_error_block_ids(self, block_ids: list[int]) -> None:
with self._invalid_block_ids_lock:
self._invalid_block_ids.update(block_ids)
def get_and_clear_block_ids_with_load_errors(self) -> set[int]:
with self._invalid_block_ids_lock:
invalid_block_ids = self._invalid_block_ids.copy()
self._invalid_block_ids.clear()
return invalid_block_ids
def _handle_request(self, req_meta: ReqMeta):
token_len = req_meta.load_spec.token_len # type: ignore[union-attr]
req_id = req_meta.req_id
@@ -737,6 +756,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
addr_list: list[list[int]] = []
size_list: list[list[int]] = []
key_list: list[str] = []
block_id_list: list[int] = []
for g_idx, db in enumerate(self.token_databases):
mask = load_mask_per_group[g_idx]
for start, end, key in db.process_tokens(
@@ -745,25 +765,29 @@ class KVCacheStoreRecvingThread(KVTransferThread):
chunk_idx = start // db.block_size
if chunk_idx >= len(mask) or not mask[chunk_idx]:
continue
addr, size, _ = db.prepare_value(start, end, req_meta.block_ids[g_idx])
addr, size, block_id = db.prepare_value(
start, end, req_meta.block_ids[g_idx]
)
key_list.append(key.to_string())
addr_list.append(addr)
size_list.append(size)
block_id_list.append(block_id)
# Rotate lists by tp_rank for load balancing
# Rotate aligned lists by tp_rank for load balancing.
rotation = self.tp_rank % len(key_list)
key_list_c = key_list[rotation:] + key_list[:rotation]
addr_list_c = addr_list[rotation:] + addr_list[:rotation]
size_list_c = size_list[rotation:] + size_list[:rotation]
key_list_c = _rotate_list(key_list, rotation)
addr_list_c = _rotate_list(addr_list, rotation)
size_list_c = _rotate_list(size_list, rotation)
block_id_list_c = _rotate_list(block_id_list, rotation)
load_batches = [(key_list_c, addr_list_c, size_list_c)]
load_batches = [(key_list_c, addr_list_c, size_list_c, block_id_list_c)]
if self.usable_disk_offload_buffer_budget_bytes is not None:
total_staging_bytes = sum(
_estimate_disk_offload_staging_bytes(size) for size in size_list_c
)
if total_staging_bytes > self.usable_disk_offload_buffer_budget_bytes:
assert self.disk_offload_buffer_budget_bytes is not None
load_batches, oversized_key = _split_disk_offload_load_batches(
split_batches, oversized_key = _split_disk_offload_load_batches(
key_list_c,
addr_list_c,
size_list_c,
@@ -772,6 +796,10 @@ class KVCacheStoreRecvingThread(KVTransferThread):
)
if oversized_key is not None:
oversized_key_index = key_list_c.index(oversized_key)
# Mark every block: we skip the whole request, and the
# tp_rank rotation means oversized_key isn't necessarily
# the first block in the request's original order.
self._add_load_error_block_ids(block_id_list_c)
oversized_key_bytes = _estimate_disk_offload_staging_bytes(
size_list_c[oversized_key_index]
)
@@ -786,12 +814,25 @@ class KVCacheStoreRecvingThread(KVTransferThread):
self.set_finished_request(req_id)
self.request_queue.task_done()
return
load_batches = []
block_id_offset = 0
for batch_keys, batch_addrs, batch_sizes in split_batches:
next_block_id_offset = block_id_offset + len(batch_keys)
batch_block_ids = block_id_list_c[
block_id_offset:next_block_id_offset
]
load_batches.append(
(batch_keys, batch_addrs, batch_sizes, batch_block_ids)
)
block_id_offset = next_block_id_offset
current_batch_keys: list[str] = key_list_c
current_batch_block_ids: list[int] = block_id_list_c
batch_bytes = 0
try:
for batch_keys, batch_addrs, batch_sizes in load_batches:
for batch_keys, batch_addrs, batch_sizes, batch_block_ids in load_batches:
current_batch_keys = batch_keys
current_batch_block_ids = batch_block_ids
batch_bytes = _sum_batch_bytes(batch_sizes)
tiers_by_key: dict[str, str] | None = None
if envs.VLLM_MOONCAKE_STORE_TIER_LOG:
@@ -806,8 +847,10 @@ class KVCacheStoreRecvingThread(KVTransferThread):
req_id, batch_keys, res, tiers_by_key
)
failed = [
(key, value)
for key, value in zip(batch_keys, res, strict=True)
(key, value, block_id)
for key, value, block_id in zip(
batch_keys, res, batch_block_ids, strict=True
)
if value < 0
]
self._record_operation(
@@ -819,15 +862,19 @@ class KVCacheStoreRecvingThread(KVTransferThread):
num_failed_keys=len(failed),
)
if failed:
self._add_load_error_block_ids(
[block_id for _, _, block_id in failed]
)
logger.warning(
"Failed to get %d Mooncake keys from sub-batch "
"(batch_keys=%d, first_failures=%s)",
len(failed),
len(batch_keys),
failed[:3],
[(key, value) for key, value, _ in failed[:3]],
)
break
except Exception as e:
self._add_load_error_block_ids(current_batch_block_ids)
self._record_operation(
"load_get",
load_get_start,
@@ -1239,6 +1286,11 @@ class MooncakeStoreWorker:
)
return done_sending, done_recving
def get_block_ids_with_load_errors(self) -> set[int]:
if self.kv_recv_thread is None:
return set()
return self.kv_recv_thread.get_and_clear_block_ids_with_load_errors()
def _record_kv_connector_operation(
self,
operation: str,