[KV Connector] Propagate MooncakeStore load failures (#42788)
Signed-off-by: Dao Le <Dao007forever@gmail.com>
This commit is contained in:
@@ -103,14 +103,18 @@ def test_worker_methods_delegate_to_store_worker():
|
||||
|
||||
worker = mock_worker_cls.return_value
|
||||
worker.get_finished.return_value = ({"req-1"}, {"req-2"})
|
||||
worker.get_block_ids_with_load_errors.return_value = {3, 4}
|
||||
connector.bind_connector_metadata(metadata)
|
||||
|
||||
connector.register_kv_caches(kv_caches)
|
||||
result = connector.get_finished(finished_req_ids)
|
||||
invalid_block_ids = connector.get_block_ids_with_load_errors()
|
||||
|
||||
worker.register_kv_caches.assert_called_once_with(kv_caches)
|
||||
worker.get_finished.assert_called_once_with(finished_req_ids, metadata)
|
||||
assert result == ({"req-1"}, {"req-2"})
|
||||
worker.get_block_ids_with_load_errors.assert_called_once_with()
|
||||
assert invalid_block_ids == {3, 4}
|
||||
|
||||
|
||||
def test_get_kv_connector_kv_cache_events_returns_none_when_empty():
|
||||
|
||||
@@ -77,6 +77,7 @@ def _make_store_sending_thread(
|
||||
def _make_store_recving_thread(
|
||||
store: MagicMock,
|
||||
*,
|
||||
tp_rank: int = 0,
|
||||
disk_offload_buffer_budget_bytes: int | None = None,
|
||||
) -> mooncake_store_worker.KVCacheStoreRecvingThread:
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
@@ -96,7 +97,7 @@ def _make_store_recving_thread(
|
||||
store=store,
|
||||
token_databases=[token_database],
|
||||
block_size=16,
|
||||
tp_rank=0,
|
||||
tp_rank=tp_rank,
|
||||
ready_event=threading.Event(),
|
||||
coord=coord,
|
||||
disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes,
|
||||
@@ -460,6 +461,67 @@ def test_store_sending_thread_only_skips_on_no_available_handle():
|
||||
assert store.batch_put_from_multi_buffers.call_count == 2
|
||||
|
||||
|
||||
def test_store_recving_thread_reports_failed_block_ids():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [256, -5, -7]
|
||||
thread = _make_store_recving_thread(store)
|
||||
|
||||
thread._handle_request(
|
||||
_make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=48,
|
||||
)
|
||||
)
|
||||
|
||||
assert thread.get_and_clear_finished_requests() == {"req-a"}
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {1, 2}
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == set()
|
||||
|
||||
|
||||
def test_store_recving_thread_reports_failed_block_ids_after_rotation():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [256, -5, 256]
|
||||
thread = _make_store_recving_thread(store, tp_rank=1)
|
||||
|
||||
thread._handle_request(
|
||||
_make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=48,
|
||||
)
|
||||
)
|
||||
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {2}
|
||||
|
||||
|
||||
def test_store_recving_thread_reports_all_attempted_blocks_on_exception():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.side_effect = RuntimeError("boom")
|
||||
thread = _make_store_recving_thread(store)
|
||||
|
||||
thread._handle_request(
|
||||
_make_load_req(
|
||||
"req-a",
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=48,
|
||||
)
|
||||
)
|
||||
|
||||
assert thread.get_and_clear_finished_requests() == {"req-a"}
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2}
|
||||
|
||||
|
||||
def test_store_worker_get_block_ids_with_load_errors_delegates_to_recv_thread():
|
||||
recv_thread = MagicMock()
|
||||
recv_thread.get_and_clear_block_ids_with_load_errors.return_value = {3, 4}
|
||||
w = _make_bare_worker()
|
||||
w.kv_recv_thread = recv_thread
|
||||
|
||||
assert w.get_block_ids_with_load_errors() == {3, 4}
|
||||
recv_thread.get_and_clear_block_ids_with_load_errors.assert_called_once_with()
|
||||
|
||||
|
||||
def test_store_sending_thread_passes_replicate_config_when_preferred_segment_set():
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
@@ -682,6 +744,7 @@ def test_recv_thread_stops_after_first_failing_disk_offload_sub_batch():
|
||||
thread._handle_request(req)
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 1
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1}
|
||||
|
||||
|
||||
def test_recv_thread_skips_split_when_budget_holds_all_keys():
|
||||
@@ -711,21 +774,26 @@ def test_recv_thread_skips_split_when_budget_holds_all_keys():
|
||||
|
||||
|
||||
def test_recv_thread_reports_unsplittable_key_larger_than_budget():
|
||||
# Non-zero tp_rank exercises the rotation: the oversized key may not sit
|
||||
# at the original request's first block. Every block must still be marked
|
||||
# invalid, since none are loaded.
|
||||
store = MagicMock()
|
||||
thread = _make_store_recving_thread(
|
||||
store,
|
||||
tp_rank=2,
|
||||
disk_offload_buffer_budget_bytes=_DISK_OFFLOAD_BUDGET_TOO_SMALL,
|
||||
)
|
||||
|
||||
req = _make_load_req(
|
||||
"req-a",
|
||||
[b"a0"],
|
||||
token_len=16,
|
||||
[b"a0", b"a1", b"a2"],
|
||||
token_len=48,
|
||||
)
|
||||
|
||||
thread._handle_request(req)
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 0
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2}
|
||||
|
||||
|
||||
def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
|
||||
|
||||
Reference in New Issue
Block a user