[KV Connector] Propagate MooncakeStore load failures (#42788)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
This commit is contained in:
Dao007forever
2026-05-25 22:12:15 -07:00
committed by GitHub
parent 7966fc7233
commit c2a4005c70
4 changed files with 143 additions and 15 deletions
@@ -103,14 +103,18 @@ def test_worker_methods_delegate_to_store_worker():
worker = mock_worker_cls.return_value
worker.get_finished.return_value = ({"req-1"}, {"req-2"})
worker.get_block_ids_with_load_errors.return_value = {3, 4}
connector.bind_connector_metadata(metadata)
connector.register_kv_caches(kv_caches)
result = connector.get_finished(finished_req_ids)
invalid_block_ids = connector.get_block_ids_with_load_errors()
worker.register_kv_caches.assert_called_once_with(kv_caches)
worker.get_finished.assert_called_once_with(finished_req_ids, metadata)
assert result == ({"req-1"}, {"req-2"})
worker.get_block_ids_with_load_errors.assert_called_once_with()
assert invalid_block_ids == {3, 4}
def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
@@ -77,6 +77,7 @@ def _make_store_sending_thread(
def _make_store_recving_thread(
store: MagicMock,
*,
tp_rank: int = 0,
disk_offload_buffer_budget_bytes: int | None = None,
) -> mooncake_store_worker.KVCacheStoreRecvingThread:
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
@@ -96,7 +97,7 @@ def _make_store_recving_thread(
store=store,
token_databases=[token_database],
block_size=16,
tp_rank=0,
tp_rank=tp_rank,
ready_event=threading.Event(),
coord=coord,
disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes,
@@ -460,6 +461,67 @@ def test_store_sending_thread_only_skips_on_no_available_handle():
assert store.batch_put_from_multi_buffers.call_count == 2
def test_store_recving_thread_reports_failed_block_ids():
store = MagicMock()
store.batch_get_into_multi_buffers.return_value = [256, -5, -7]
thread = _make_store_recving_thread(store)
thread._handle_request(
_make_load_req(
"req-a",
[b"a0", b"a1", b"a2"],
token_len=48,
)
)
assert thread.get_and_clear_finished_requests() == {"req-a"}
assert thread.get_and_clear_block_ids_with_load_errors() == {1, 2}
assert thread.get_and_clear_block_ids_with_load_errors() == set()
def test_store_recving_thread_reports_failed_block_ids_after_rotation():
store = MagicMock()
store.batch_get_into_multi_buffers.return_value = [256, -5, 256]
thread = _make_store_recving_thread(store, tp_rank=1)
thread._handle_request(
_make_load_req(
"req-a",
[b"a0", b"a1", b"a2"],
token_len=48,
)
)
assert thread.get_and_clear_block_ids_with_load_errors() == {2}
def test_store_recving_thread_reports_all_attempted_blocks_on_exception():
store = MagicMock()
store.batch_get_into_multi_buffers.side_effect = RuntimeError("boom")
thread = _make_store_recving_thread(store)
thread._handle_request(
_make_load_req(
"req-a",
[b"a0", b"a1", b"a2"],
token_len=48,
)
)
assert thread.get_and_clear_finished_requests() == {"req-a"}
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2}
def test_store_worker_get_block_ids_with_load_errors_delegates_to_recv_thread():
recv_thread = MagicMock()
recv_thread.get_and_clear_block_ids_with_load_errors.return_value = {3, 4}
w = _make_bare_worker()
w.kv_recv_thread = recv_thread
assert w.get_block_ids_with_load_errors() == {3, 4}
recv_thread.get_and_clear_block_ids_with_load_errors.assert_called_once_with()
def test_store_sending_thread_passes_replicate_config_when_preferred_segment_set():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
@@ -682,6 +744,7 @@ def test_recv_thread_stops_after_first_failing_disk_offload_sub_batch():
thread._handle_request(req)
assert store.batch_get_into_multi_buffers.call_count == 1
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1}
def test_recv_thread_skips_split_when_budget_holds_all_keys():
@@ -711,21 +774,26 @@ def test_recv_thread_skips_split_when_budget_holds_all_keys():
def test_recv_thread_reports_unsplittable_key_larger_than_budget():
# Non-zero tp_rank exercises the rotation: the oversized key may not sit
# at the original request's first block. Every block must still be marked
# invalid, since none are loaded.
store = MagicMock()
thread = _make_store_recving_thread(
store,
tp_rank=2,
disk_offload_buffer_budget_bytes=_DISK_OFFLOAD_BUDGET_TOO_SMALL,
)
req = _make_load_req(
"req-a",
[b"a0"],
token_len=16,
[b"a0", b"a1", b"a2"],
token_len=48,
)
thread._handle_request(req)
assert store.batch_get_into_multi_buffers.call_count == 0
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2}
def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):