[KV Connector] Propagate MooncakeStore load failures (#42788)
Signed-off-by: Dao Le <Dao007forever@gmail.com>
This commit is contained in:
@@ -103,14 +103,18 @@ def test_worker_methods_delegate_to_store_worker():
|
||||
|
||||
worker = mock_worker_cls.return_value
|
||||
worker.get_finished.return_value = ({"req-1"}, {"req-2"})
|
||||
worker.get_block_ids_with_load_errors.return_value = {3, 4}
|
||||
connector.bind_connector_metadata(metadata)
|
||||
|
||||
connector.register_kv_caches(kv_caches)
|
||||
result = connector.get_finished(finished_req_ids)
|
||||
invalid_block_ids = connector.get_block_ids_with_load_errors()
|
||||
|
||||
worker.register_kv_caches.assert_called_once_with(kv_caches)
|
||||
worker.get_finished.assert_called_once_with(finished_req_ids, metadata)
|
||||
assert result == ({"req-1"}, {"req-2"})
|
||||
worker.get_block_ids_with_load_errors.assert_called_once_with()
|
||||
assert invalid_block_ids == {3, 4}
|
||||
|
||||
|
||||
def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
|
||||
|
||||
@@ -77,6 +77,7 @@ def _make_store_sending_thread(
|
||||
def _make_store_recving_thread(
|
||||
store: MagicMock,
|
||||
*,
|
||||
tp_rank: int = 0,
|
||||
disk_offload_buffer_budget_bytes: int | None = None,
|
||||
) -> mooncake_store_worker.KVCacheStoreRecvingThread:
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
@@ -96,7 +97,7 @@ def _make_store_recving_thread(
|
||||
store=store,
|
||||
token_databases=[token_database],
|
||||
block_size=16,
|
||||
tp_rank=0,
|
||||
tp_rank=tp_rank,
|
||||
ready_event=threading.Event(),
|
||||
coord=coord,
|
||||
disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes,
|
||||
@@ -460,6 +461,67 @@ def test_store_sending_thread_only_skips_on_no_available_handle():
|
||||
assert store.batch_put_from_multi_buffers.call_count == 2
|
||||
|
||||
|
||||
def test_store_recving_thread_reports_failed_block_ids():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [256, -5, -7]
|
||||
thread = _make_store_recving_thread(store)
|
||||
|
||||
thread._handle_request(
|
||||
_make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=48,
|
||||
)
|
||||
)
|
||||
|
||||
assert thread.get_and_clear_finished_requests() == {"req-a"}
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {1, 2}
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == set()
|
||||
|
||||
|
||||
def test_store_recving_thread_reports_failed_block_ids_after_rotation():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [256, -5, 256]
|
||||
thread = _make_store_recving_thread(store, tp_rank=1)
|
||||
|
||||
thread._handle_request(
|
||||
_make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=48,
|
||||
)
|
||||
)
|
||||
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {2}
|
||||
|
||||
|
||||
def test_store_recving_thread_reports_all_attempted_blocks_on_exception():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.side_effect = RuntimeError("boom")
|
||||
thread = _make_store_recving_thread(store)
|
||||
|
||||
thread._handle_request(
|
||||
_make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=48,
|
||||
)
|
||||
)
|
||||
|
||||
assert thread.get_and_clear_finished_requests() == {"req-a"}
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2}
|
||||
|
||||
|
||||
def test_store_worker_get_block_ids_with_load_errors_delegates_to_recv_thread():
|
||||
recv_thread = MagicMock()
|
||||
recv_thread.get_and_clear_block_ids_with_load_errors.return_value = {3, 4}
|
||||
w = _make_bare_worker()
|
||||
w.kv_recv_thread = recv_thread
|
||||
|
||||
assert w.get_block_ids_with_load_errors() == {3, 4}
|
||||
recv_thread.get_and_clear_block_ids_with_load_errors.assert_called_once_with()
|
||||
|
||||
|
||||
def test_store_sending_thread_passes_replicate_config_when_preferred_segment_set():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
@@ -682,6 +744,7 @@ def test_recv_thread_stops_after_first_failing_disk_offload_sub_batch():
|
||||
thread._handle_request(req)
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 1
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1}
|
||||
|
||||
|
||||
def test_recv_thread_skips_split_when_budget_holds_all_keys():
|
||||
@@ -711,21 +774,26 @@ def test_recv_thread_skips_split_when_budget_holds_all_keys():
|
||||
|
||||
|
||||
def test_recv_thread_reports_unsplittable_key_larger_than_budget():
|
||||
# Non-zero tp_rank exercises the rotation: the oversized key may not sit
|
||||
# at the original request's first block. Every block must still be marked
|
||||
# invalid, since none are loaded.
|
||||
store = MagicMock()
|
||||
thread = _make_store_recving_thread(
|
||||
store,
|
||||
tp_rank=2,
|
||||
disk_offload_buffer_budget_bytes=_DISK_OFFLOAD_BUDGET_TOO_SMALL,
|
||||
)
|
||||
|
||||
req = _make_load_req(
|
||||
"req-a",
|
||||
[b"a0"],
|
||||
token_len=16,
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=48,
|
||||
)
|
||||
|
||||
thread._handle_request(req)
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 0
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2}
|
||||
|
||||
|
||||
def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
|
||||
|
||||
@@ -270,6 +270,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
return self.connector_worker.get_finished(finished_req_ids, metadata)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_block_ids_with_load_errors()
|
||||
|
||||
def get_kv_connector_kv_cache_events(
|
||||
self,
|
||||
) -> MooncakeStoreKVEvents | None:
|
||||
|
||||
@@ -20,7 +20,7 @@ import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal
|
||||
from typing import Any, Literal, TypeVar
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
@@ -68,6 +68,12 @@ DEFAULT_GLOBAL_SEGMENT_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
|
||||
DEFAULT_LOCAL_BUFFER_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
|
||||
|
||||
MOONCAKE_NO_AVAILABLE_HANDLE = -200
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
def _rotate_list(values: list[_T], offset: int) -> list[_T]:
|
||||
return values[offset:] + values[:offset]
|
||||
|
||||
|
||||
# Mirrors FileStorageConfig::local_buffer_size in Mooncake C++.
|
||||
DEFAULT_MOONCAKE_DISK_STAGING_BUFFER_BYTES = 1280 * 1024 * 1024
|
||||
@@ -711,6 +717,9 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
name="KVCacheStoreRecvingThread",
|
||||
record_operation=record_operation,
|
||||
)
|
||||
# _invalid_block_ids can be access by both the Worker and RecvingThread
|
||||
self._invalid_block_ids_lock = threading.Lock()
|
||||
self._invalid_block_ids: set[int] = set()
|
||||
self.disk_offload_buffer_budget_bytes = disk_offload_buffer_budget_bytes
|
||||
self.usable_disk_offload_buffer_budget_bytes = (
|
||||
None
|
||||
@@ -721,6 +730,16 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
)
|
||||
self.coord = coord
|
||||
|
||||
def _add_load_error_block_ids(self, block_ids: list[int]) -> None:
|
||||
with self._invalid_block_ids_lock:
|
||||
self._invalid_block_ids.update(block_ids)
|
||||
|
||||
def get_and_clear_block_ids_with_load_errors(self) -> set[int]:
|
||||
with self._invalid_block_ids_lock:
|
||||
invalid_block_ids = self._invalid_block_ids.copy()
|
||||
self._invalid_block_ids.clear()
|
||||
return invalid_block_ids
|
||||
|
||||
def _handle_request(self, req_meta: ReqMeta):
|
||||
token_len = req_meta.load_spec.token_len # type: ignore[union-attr]
|
||||
req_id = req_meta.req_id
|
||||
@@ -737,6 +756,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
addr_list: list[list[int]] = []
|
||||
size_list: list[list[int]] = []
|
||||
key_list: list[str] = []
|
||||
block_id_list: list[int] = []
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
mask = load_mask_per_group[g_idx]
|
||||
for start, end, key in db.process_tokens(
|
||||
@@ -745,25 +765,29 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
chunk_idx = start // db.block_size
|
||||
if chunk_idx >= len(mask) or not mask[chunk_idx]:
|
||||
continue
|
||||
addr, size, _ = db.prepare_value(start, end, req_meta.block_ids[g_idx])
|
||||
addr, size, block_id = db.prepare_value(
|
||||
start, end, req_meta.block_ids[g_idx]
|
||||
)
|
||||
key_list.append(key.to_string())
|
||||
addr_list.append(addr)
|
||||
size_list.append(size)
|
||||
block_id_list.append(block_id)
|
||||
|
||||
# Rotate lists by tp_rank for load balancing
|
||||
# Rotate aligned lists by tp_rank for load balancing.
|
||||
rotation = self.tp_rank % len(key_list)
|
||||
key_list_c = key_list[rotation:] + key_list[:rotation]
|
||||
addr_list_c = addr_list[rotation:] + addr_list[:rotation]
|
||||
size_list_c = size_list[rotation:] + size_list[:rotation]
|
||||
key_list_c = _rotate_list(key_list, rotation)
|
||||
addr_list_c = _rotate_list(addr_list, rotation)
|
||||
size_list_c = _rotate_list(size_list, rotation)
|
||||
block_id_list_c = _rotate_list(block_id_list, rotation)
|
||||
|
||||
load_batches = [(key_list_c, addr_list_c, size_list_c)]
|
||||
load_batches = [(key_list_c, addr_list_c, size_list_c, block_id_list_c)]
|
||||
if self.usable_disk_offload_buffer_budget_bytes is not None:
|
||||
total_staging_bytes = sum(
|
||||
_estimate_disk_offload_staging_bytes(size) for size in size_list_c
|
||||
)
|
||||
if total_staging_bytes > self.usable_disk_offload_buffer_budget_bytes:
|
||||
assert self.disk_offload_buffer_budget_bytes is not None
|
||||
load_batches, oversized_key = _split_disk_offload_load_batches(
|
||||
split_batches, oversized_key = _split_disk_offload_load_batches(
|
||||
key_list_c,
|
||||
addr_list_c,
|
||||
size_list_c,
|
||||
@@ -772,6 +796,10 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
)
|
||||
if oversized_key is not None:
|
||||
oversized_key_index = key_list_c.index(oversized_key)
|
||||
# Mark every block: we skip the whole request, and the
|
||||
# tp_rank rotation means oversized_key isn't necessarily
|
||||
# the first block in the request's original order.
|
||||
self._add_load_error_block_ids(block_id_list_c)
|
||||
oversized_key_bytes = _estimate_disk_offload_staging_bytes(
|
||||
size_list_c[oversized_key_index]
|
||||
)
|
||||
@@ -786,12 +814,25 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
self.set_finished_request(req_id)
|
||||
self.request_queue.task_done()
|
||||
return
|
||||
load_batches = []
|
||||
block_id_offset = 0
|
||||
for batch_keys, batch_addrs, batch_sizes in split_batches:
|
||||
next_block_id_offset = block_id_offset + len(batch_keys)
|
||||
batch_block_ids = block_id_list_c[
|
||||
block_id_offset:next_block_id_offset
|
||||
]
|
||||
load_batches.append(
|
||||
(batch_keys, batch_addrs, batch_sizes, batch_block_ids)
|
||||
)
|
||||
block_id_offset = next_block_id_offset
|
||||
|
||||
current_batch_keys: list[str] = key_list_c
|
||||
current_batch_block_ids: list[int] = block_id_list_c
|
||||
batch_bytes = 0
|
||||
try:
|
||||
for batch_keys, batch_addrs, batch_sizes in load_batches:
|
||||
for batch_keys, batch_addrs, batch_sizes, batch_block_ids in load_batches:
|
||||
current_batch_keys = batch_keys
|
||||
current_batch_block_ids = batch_block_ids
|
||||
batch_bytes = _sum_batch_bytes(batch_sizes)
|
||||
tiers_by_key: dict[str, str] | None = None
|
||||
if envs.VLLM_MOONCAKE_STORE_TIER_LOG:
|
||||
@@ -806,8 +847,10 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
req_id, batch_keys, res, tiers_by_key
|
||||
)
|
||||
failed = [
|
||||
(key, value)
|
||||
for key, value in zip(batch_keys, res, strict=True)
|
||||
(key, value, block_id)
|
||||
for key, value, block_id in zip(
|
||||
batch_keys, res, batch_block_ids, strict=True
|
||||
)
|
||||
if value < 0
|
||||
]
|
||||
self._record_operation(
|
||||
@@ -819,15 +862,19 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
num_failed_keys=len(failed),
|
||||
)
|
||||
if failed:
|
||||
self._add_load_error_block_ids(
|
||||
[block_id for _, _, block_id in failed]
|
||||
)
|
||||
logger.warning(
|
||||
"Failed to get %d Mooncake keys from sub-batch "
|
||||
"(batch_keys=%d, first_failures=%s)",
|
||||
len(failed),
|
||||
len(batch_keys),
|
||||
failed[:3],
|
||||
[(key, value) for key, value, _ in failed[:3]],
|
||||
)
|
||||
break
|
||||
except Exception as e:
|
||||
self._add_load_error_block_ids(current_batch_block_ids)
|
||||
self._record_operation(
|
||||
"load_get",
|
||||
load_get_start,
|
||||
@@ -1239,6 +1286,11 @@ class MooncakeStoreWorker:
|
||||
)
|
||||
return done_sending, done_recving
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
if self.kv_recv_thread is None:
|
||||
return set()
|
||||
return self.kv_recv_thread.get_and_clear_block_ids_with_load_errors()
|
||||
|
||||
def _record_kv_connector_operation(
|
||||
self,
|
||||
operation: str,
|
||||
|
||||
Reference in New Issue
Block a user