From 8b1d188046034b42024d22ee52fa6c3948b4acbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Wed, 16 Sep 2026 14:55:40 +0200 Subject: [PATCH] [Bugfix][Mooncake] Report request-level KV load failures under HMA (#56855) Signed-off-by: NickLucche --- .../unit/test_kv_load_failure_recovery.py | 345 +----------------- .../unit/test_mooncake_connector_hma.py | 77 ++++ .../unit/test_mooncake_store_connector.py | 33 ++ .../unit/test_mooncake_store_worker.py | 82 +++++ tests/v1/kv_connector/unit/utils.py | 2 - .../v1/mooncake/mooncake_connector.py | 46 ++- .../v1/mooncake/store/connector.py | 9 + .../kv_connector/v1/mooncake/store/worker.py | 60 ++- vllm/v1/core/sched/scheduler.py | 52 +-- 9 files changed, 329 insertions(+), 377 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py b/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py index 74bac55598..a54a023a13 100644 --- a/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py +++ b/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py @@ -2,21 +2,12 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Callable -from typing import Literal -from unittest.mock import Mock, patch +from unittest.mock import Mock import pytest -import torch -from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.core.sched.scheduler import Scheduler -from vllm.v1.kv_cache_interface import ( - KVCacheConfig, - KVCacheGroupSpec, - MLAAttentionSpec, - SlidingWindowMLASpec, -) -from vllm.v1.request import FinishReason, Request, RequestStatus +from vllm.v1.request import Request, RequestStatus from .utils import ( create_model_runner_output, @@ -44,18 +35,6 @@ def scheduler(): return create_scheduler(vllm_config) -@pytest.fixture -def hybrid_scheduler(): - """Scheduler for a hybrid model: full attention plus a Mamba group.""" - vllm_config = create_vllm_config(kv_load_failure_policy="recompute") - kv_cache_config = make_kv_cache_config( - block_size=vllm_config.cache_config.block_size, - mamba_enabled=True, - num_blocks=10000, - ) - return create_scheduler(vllm_config, kv_cache_config=kv_cache_config) - - @pytest.mark.parametrize( "num_prompt_blocks,num_external_computed_blocks,invalid_block_idxs", [ @@ -361,317 +340,19 @@ def test_async_progressive_load_failure( assert scheduler.connector.get_num_new_matched_tokens.call_count == 1 -def _schedule_hybrid_async_load( - scheduler: Scheduler, num_prompt_blocks: int, num_external_computed_blocks: int -) -> Request: - """Put one request into an async external load on a hybrid model.""" - request = create_request(num_tokens=num_prompt_blocks * scheduler.block_size) - scheduler.add_request(request=request) +@pytest.mark.cpu_test +def test_block_level_load_failure_fails_closed_with_multiple_groups(): + """Block-level invalid-block reporting is rejected on multi-group layouts. - scheduler.connector = Mock() - scheduler.connector.get_num_new_matched_tokens.side_effect = ( - _make_get_num_new_matched_tokens( - {request.request_id: num_external_computed_blocks * scheduler.block_size}, - async_load=True, - ) - ) - scheduler.connector.take_events.return_value = () - return request - - -def _report_invalid_blocks( - scheduler: Scheduler, scheduler_output, invalid_block_ids: set[int] -) -> None: - scheduler.update_from_output( - scheduler_output, - create_model_runner_output( - reqs=[], - finished_recving=set(), - invalid_block_ids=invalid_block_ids, - use_eos=True, - ), - ) - - -@pytest.mark.parametrize("failed_group_idx", [0, 1]) -def test_hybrid_load_failure_recomputes_whole_request( - hybrid_scheduler: Scheduler, failed_group_idx: int -): - """A hybrid model must recover, not crash. - - Its groups sit on different block grids, so there is no single truncation - point; the request restarts from scratch instead. + Block IDs are only unique within a group, so the scheduler cannot map + them back to requests; connectors must report failed_recving instead. """ - num_prompt_blocks, num_external_computed_blocks = 10, 9 - request = _schedule_hybrid_async_load( - hybrid_scheduler, num_prompt_blocks, num_external_computed_blocks - ) - scheduler_output = hybrid_scheduler.schedule() - - block_ids_per_group = hybrid_scheduler.kv_cache_manager.get_block_ids( - request.request_id - ) - assert len(block_ids_per_group) == 2, "expected a hybrid (multi-group) layout" - null_block_id = hybrid_scheduler.kv_cache_manager.block_pool.null_block.block_id - # The Mamba group backs only its aligned snapshot with a real block, so - # pick a block that actually belongs to this request. - failed_block = next( - block_id - for block_id in block_ids_per_group[failed_group_idx] - if block_id != null_block_id - ) - - _report_invalid_blocks(hybrid_scheduler, scheduler_output, {failed_block}) - - assert request.num_computed_tokens == 0 - assert hybrid_scheduler.failed_recving_kv_req_ids == {request.request_id} - assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS - - -def test_hybrid_load_failure_ignores_null_block(hybrid_scheduler: Scheduler): - """The null block is shared by every request and holds no KV, so reporting - it must not restart anyone.""" - num_prompt_blocks, num_external_computed_blocks = 10, 9 - request = _schedule_hybrid_async_load( - hybrid_scheduler, num_prompt_blocks, num_external_computed_blocks - ) - scheduler_output = hybrid_scheduler.schedule() - - null_block_id = hybrid_scheduler.kv_cache_manager.block_pool.null_block.block_id - _, mamba_block_ids = hybrid_scheduler.kv_cache_manager.get_block_ids( - request.request_id - ) - # The Mamba group parks its unaligned positions on the null block, so a - # request that never failed would otherwise be restarted by this report. - assert mamba_block_ids[0] == null_block_id - - num_computed_tokens = request.num_computed_tokens - _report_invalid_blocks(hybrid_scheduler, scheduler_output, {null_block_id}) - - assert request.num_computed_tokens == num_computed_tokens - assert not hybrid_scheduler.failed_recving_kv_req_ids - - -def _make_deepseek_v4_five_group_config(num_blocks: int) -> KVCacheConfig: - """Build the scheduler-side projection of V4-Flash's packed KV layout.""" - - def mla_spec() -> MLAAttentionSpec: - return MLAAttentionSpec( - block_size=256, - num_kv_heads=1, - head_size=512, - dtype=torch.bfloat16, - tokens_per_state=4, - model_version="deepseek_v4", - ) - - def swa_spec( - *, block_size: int, sliding_window: int, head_size: int - ) -> SlidingWindowMLASpec: - return SlidingWindowMLASpec( - block_size=block_size, - num_kv_heads=1, - head_size=head_size, - dtype=torch.float32, - sliding_window=sliding_window, - model_version="deepseek_v4", - ) - - # DeepSeek-V4 packs these into five physical groups. The two SWA groups - # have identical scheduling semantics but are split to align layer tuples. - groups = [ - KVCacheGroupSpec(["full_mla_and_indexer"], mla_spec()), - KVCacheGroupSpec( - ["c4_state"], - swa_spec(block_size=4, sliding_window=8, head_size=2048), - ), - KVCacheGroupSpec( - ["c128_state"], - swa_spec(block_size=8, sliding_window=128, head_size=2048), - ), - KVCacheGroupSpec( - ["swa.0"], - swa_spec(block_size=64, sliding_window=512, head_size=512), - ), - KVCacheGroupSpec( - ["swa.1"], - swa_spec(block_size=64, sliding_window=512, head_size=512), - ), - ] - return KVCacheConfig( - num_blocks=num_blocks, - kv_cache_tensors=[], - kv_cache_groups=groups, - ) - - -def _create_deepseek_v4_scheduler( - kv_load_failure_policy: Literal["recompute", "fail"] = "recompute", -) -> Scheduler: - num_blocks = 10000 - vllm_config = create_vllm_config( - block_size=256, - max_num_batched_tokens=4096, - max_model_len=8192, - # max_model_len only reaches SchedulerConfig; opt-125m still caps the - # model at 2048, which is below the 9-block requests used here. - hf_overrides={"max_position_embeddings": 8192}, - kv_load_failure_policy=kv_load_failure_policy, - ) - return create_scheduler( + vllm_config = create_vllm_config() + scheduler = create_scheduler( vllm_config, - num_blocks=num_blocks, - kv_cache_config=_make_deepseek_v4_five_group_config(num_blocks), - hash_block_size=4, + kv_cache_config=make_kv_cache_config(block_size=16, swa_enabled=True), ) + assert len(scheduler.kv_cache_config.kv_cache_groups) == 2 - -@pytest.fixture -def deepseek_v4_scheduler() -> Scheduler: - return _create_deepseek_v4_scheduler() - - -def _schedule_deepseek_v4_async_load( - scheduler: Scheduler, -) -> tuple[Request, SchedulerOutput]: - num_external_tokens = 8 * scheduler.block_size - request = create_request(num_tokens=9 * scheduler.block_size, block_size=4) - scheduler.add_request(request) - - scheduler.connector = Mock() - scheduler.connector.get_num_new_matched_tokens.side_effect = ( - _make_get_num_new_matched_tokens( - {request.request_id: num_external_tokens}, async_load=True - ) - ) - scheduler.connector.take_events.return_value = () - return request, scheduler.schedule() - - -@pytest.mark.parametrize("failed_group_idx", range(5)) -def test_deepseek_v4_load_failure_recomputes_whole_request( - deepseek_v4_scheduler: Scheduler, failed_group_idx: int -): - """A failed load in any V4-Flash KV group must not kill EngineCore.""" - request, scheduler_output = _schedule_deepseek_v4_async_load(deepseek_v4_scheduler) - - block_ids_per_group = deepseek_v4_scheduler.kv_cache_manager.get_block_ids( - request.request_id - ) - assert len(block_ids_per_group) == 5 - null_block_id = ( - deepseek_v4_scheduler.kv_cache_manager.block_pool.null_block.block_id - ) - failed_block_id = next( - block_id - for block_id in block_ids_per_group[failed_group_idx] - if block_id != null_block_id - ) - - deepseek_v4_scheduler.update_from_output( - scheduler_output, - create_model_runner_output( - reqs=[], - finished_recving=set(), - invalid_block_ids={failed_block_id}, - use_eos=True, - ), - ) - - assert request.num_computed_tokens == 0 - assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS - assert deepseek_v4_scheduler.failed_recving_kv_req_ids == {request.request_id} - - -def test_deepseek_v4_load_failure_ignores_shared_null_block( - deepseek_v4_scheduler: Scheduler, -): - """The shared null block contains no request KV and cannot fail a load.""" - request, scheduler_output = _schedule_deepseek_v4_async_load(deepseek_v4_scheduler) - null_block_id = ( - deepseek_v4_scheduler.kv_cache_manager.block_pool.null_block.block_id - ) - block_ids_per_group = deepseek_v4_scheduler.kv_cache_manager.get_block_ids( - request.request_id - ) - assert any(null_block_id in group for group in block_ids_per_group[1:]) - num_computed_tokens = request.num_computed_tokens - - deepseek_v4_scheduler.update_from_output( - scheduler_output, - create_model_runner_output( - reqs=[], - finished_recving=set(), - invalid_block_ids={null_block_id}, - use_eos=True, - ), - ) - - assert request.num_computed_tokens == num_computed_tokens - assert not deepseek_v4_scheduler.failed_recving_kv_req_ids - - -def test_deepseek_v4_sync_load_failure_fails_and_evicts_real_blocks(): - """Fail policy must terminate cleanly and never evict the null block.""" - scheduler = _create_deepseek_v4_scheduler(kv_load_failure_policy="fail") - num_external_tokens = 8 * scheduler.block_size - request = create_request(num_tokens=9 * scheduler.block_size, block_size=4) - scheduler.add_request(request) - - scheduler.connector = Mock() - scheduler.connector.get_num_new_matched_tokens.side_effect = ( - _make_get_num_new_matched_tokens( - {request.request_id: num_external_tokens}, async_load=False - ) - ) - scheduler.connector.request_finished.return_value = (False, None) - scheduler.connector.take_events.return_value = () - scheduler_output = scheduler.schedule() - - block_ids_per_group = scheduler.kv_cache_manager.get_block_ids(request.request_id) - assert len(block_ids_per_group) == 5 - null_block_id = scheduler.kv_cache_manager.block_pool.null_block.block_id - real_block_ids = { - block_id - for group_block_ids in block_ids_per_group - for block_id in group_block_ids - if block_id != null_block_id - } - failed_block_id = next( - block_id for block_id in block_ids_per_group[2] if block_id != null_block_id - ) - - evicted_block_ids: list[set[int]] = [] - original_evict_blocks = scheduler.kv_cache_manager.evict_blocks - - def evict_blocks_spy(block_ids: set[int]) -> None: - evicted_block_ids.append(set(block_ids)) - original_evict_blocks(block_ids) - - with ( - patch.object( - scheduler.kv_cache_manager, - "evict_blocks", - side_effect=evict_blocks_spy, - ), - patch.object(scheduler, "_connector_finished", return_value=(False, None)), - ): - outputs = scheduler.update_from_output( - scheduler_output, - create_model_runner_output( - [request], - invalid_block_ids={failed_block_id}, - use_eos=True, - ), - ) - - assert request.status == RequestStatus.FINISHED_ERROR - assert request.get_finished_reason() == FinishReason.ERROR - assert request.request_id not in scheduler.requests - assert evicted_block_ids == [real_block_ids] - assert null_block_id not in evicted_block_ids[0] - - engine_outputs = next(iter(outputs.values())) - assert len(engine_outputs.outputs) == 1 - assert engine_outputs.outputs[0].request_id == request.request_id - assert engine_outputs.outputs[0].finish_reason == FinishReason.ERROR + with pytest.raises(RuntimeError, match="failed_recving"): + scheduler._handle_invalid_blocks({1}, {}) diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py b/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py index 54b97d482e..f3739ef430 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py @@ -18,6 +18,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im MooncakeConnectorMetadata, MooncakeConnectorScheduler, MooncakeXferMetadata, + PullReqMeta, SendBlockMeta, TransferRegion, ) @@ -436,3 +437,79 @@ def test_request_finished_with_hma_groups(): assert stored_blocks[0] == fa_blocks # SW: clipped to last 9 blocks (sw_size=128, block_size=16 → 8+1=9) assert stored_blocks[1] == sw_blocks[-9:] + + +# --------------------------------------------------------------------------- +# Worker-side load-failure reporting (HMA vs non-HMA) +# --------------------------------------------------------------------------- +def _make_kv_consumer_worker(swa_enabled: bool, disable_hma: bool = False): + block_size = 16 + vllm_config = create_vllm_config( + kv_connector="MooncakeConnector", + kv_role="kv_consumer", + block_size=block_size, + ) + vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma + kv_cache_config = make_kv_cache_config( + block_size=block_size, swa_enabled=swa_enabled + ) + with set_current_vllm_config(vllm_config), patch_worker_dependencies(): + connector = MooncakeConnector( + vllm_config, KVConnectorRole.WORKER, kv_cache_config + ) + return connector.connector_worker + + +def _make_pull_meta(d_req_id: str, local_block_ids: list[list[int]]) -> PullReqMeta: + return PullReqMeta( + d_req_id=d_req_id, + transfer_id="tx-1", + local_block_ids=local_block_ids, + remote_engine_id="engine-0", + remote_bootstrap_addr="127.0.0.1:1234", + ) + + +# The tests below construct a MooncakeConnectorWorker, whose __init__ +# requires an accelerator device, so (like the worker tests in +# test_mooncake_connector.py) they run on the GPU KV-connectors job only. +@pytest.mark.parametrize( + "swa_enabled,disable_hma,expected_is_hma", + [ + (True, False, True), # SWA group present, HMA enabled + (True, True, False), # SWA group present, but HMA disabled + (False, False, False), # FA only, HMA not needed + ], +) +def test_worker_is_hma_required(swa_enabled, disable_hma, expected_is_hma): + """Worker-side _is_hma_required mirrors the scheduler's derivation.""" + worker = _make_kv_consumer_worker(swa_enabled, disable_hma) + assert worker._is_hma_required is expected_is_hma + + +def test_worker_failed_recv_reports_request_level_failure_with_hma(): + """With HMA, load failures report the request, not ambiguous block IDs.""" + worker = _make_kv_consumer_worker(swa_enabled=True) + assert worker._is_hma_required + + pull_meta = _make_pull_meta("d-req-1", [[1, 2], [3, 4]]) + worker._handle_failed_recv({"p-req-1": pull_meta}, ["p-req-1"], "boom") + + assert worker.get_block_ids_with_load_errors() == set() + results = worker.get_transfer_results() + assert results.failed_recving == {"d-req-1"} + assert results.finished_recving == {"d-req-1"} + + +def test_worker_failed_recv_reports_block_ids_without_hma(): + """Without HMA, load failures keep reporting block-level errors.""" + worker = _make_kv_consumer_worker(swa_enabled=False) + assert not worker._is_hma_required + + pull_meta = _make_pull_meta("d-req-1", [[1, 2], [3, 4]]) + worker._handle_failed_recv({"p-req-1": pull_meta}, ["p-req-1"], "boom") + + assert worker.get_block_ids_with_load_errors() == {1, 2, 3, 4} + results = worker.get_transfer_results() + assert results.failed_recving == set() + assert results.finished_recving == {"d-req-1"} diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_connector.py b/tests/v1/kv_connector/unit/test_mooncake_store_connector.py index 6afb529d14..f3ffa55a54 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_connector.py @@ -12,6 +12,7 @@ from vllm.config import set_current_vllm_config from vllm.distributed.kv_events import BlockStored from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorRole, + KVConnectorTransferResults, ) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import ( connector as mooncake_store_connector, @@ -245,6 +246,38 @@ def test_worker_methods_delegate_to_store_worker(): assert invalid_block_ids == {3, 4} +def test_get_transfer_results_delegates_to_store_worker(): + vllm_config = _make_vllm_config() + kv_cache_config = _make_kv_cache_config() + metadata = MooncakeStoreConnectorMetadata(set(), set()) + finished_req_ids = {"req-1"} + + with ( + set_current_vllm_config(vllm_config), + patch( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store." + "connector.MooncakeStoreWorker" + ) as mock_worker_cls, + ): + connector = mooncake_store_connector.MooncakeStoreConnector( + vllm_config, KVConnectorRole.WORKER, kv_cache_config + ) + + worker = mock_worker_cls.return_value + transfer_results = KVConnectorTransferResults( + finished_sending={"req-1"}, + finished_recving={"req-2"}, + failed_recving={"req-3"}, + ) + worker.get_transfer_results.return_value = transfer_results + connector.bind_connector_metadata(metadata) + + result = connector.get_transfer_results(finished_req_ids) + + worker.get_transfer_results.assert_called_once_with(finished_req_ids, metadata) + assert result is transfer_results + + def test_get_kv_connector_kv_cache_events_returns_none_when_disabled(): vllm_config = _make_vllm_config() kv_cache_config = _make_kv_cache_config() diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 9baf16e261..dea4178ff3 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -177,6 +177,7 @@ def _make_store_recving_thread( *, tp_rank: int = 0, disk_offload_buffer_budget_bytes: int | None = None, + is_hma_required: bool = False, ) -> mooncake_store_worker.KVCacheStoreRecvingThread: from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec @@ -199,6 +200,7 @@ def _make_store_recving_thread( ready_event=threading.Event(), coord=coord, disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes, + is_hma_required=is_hma_required, ) thread.request_queue.task_done = MagicMock() return thread @@ -303,6 +305,7 @@ def _make_vllm_config( kv_role: str = "kv_both", pipeline_parallel_size: int = 1, kv_cache_layout: KVCacheLayout = KVCacheLayout.LBHNC, + disable_hybrid_kv_cache_manager: bool = True, ) -> SimpleNamespace: cache_config = SimpleNamespace(block_size=16, num_gpu_blocks=10) cache_config.get_resolved_kv_cache_layout = lambda: kv_cache_layout @@ -319,6 +322,9 @@ def _make_vllm_config( kv_role=kv_role, extra_config=extra_config ), cache_config=cache_config, + scheduler_config=SimpleNamespace( + disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager + ), kv_events_config=SimpleNamespace(enable_kv_cache_events=False), speculative_config=None, ) @@ -2147,6 +2153,48 @@ def test_recv_thread_reports_unsplittable_key_larger_than_budget(): assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2} +def test_recv_thread_partial_load_failure_reports_block_ids_without_hma(): + store = MagicMock() + # First key fails, second succeeds. + store.batch_get_into_multi_buffers.return_value = [-1, 0] + thread = _make_store_recving_thread(store) + + thread._handle_request(_make_load_req("req-a", [b"h0", b"h1"], token_len=32)) + + assert thread.get_and_clear_block_ids_with_load_errors() == {0} + assert thread.get_and_clear_failed_requests() == set() + assert thread.get_and_clear_finished_requests() == {"req-a"} + + +def test_recv_thread_partial_load_failure_reports_request_with_hma(): + store = MagicMock() + store.batch_get_into_multi_buffers.return_value = [-1, 0] + thread = _make_store_recving_thread(store, is_hma_required=True) + + thread._handle_request(_make_load_req("req-a", [b"h0", b"h1"], token_len=32)) + + assert thread.get_and_clear_block_ids_with_load_errors() == set() + assert thread.get_and_clear_failed_requests() == {"req-a"} + assert thread.get_and_clear_finished_requests() == {"req-a"} + + +def test_recv_thread_oversized_key_reports_request_with_hma(): + store = MagicMock() + thread = _make_store_recving_thread( + store, + tp_rank=2, + disk_offload_buffer_budget_bytes=_DISK_OFFLOAD_BUDGET_TOO_SMALL, + is_hma_required=True, + ) + + thread._handle_request(_make_load_req("req-a", [b"a0", b"a1", b"a2"], token_len=48)) + + assert store.batch_get_into_multi_buffers.call_count == 0 + assert thread.get_and_clear_block_ids_with_load_errors() == set() + assert thread.get_and_clear_failed_requests() == {"req-a"} + assert thread.get_and_clear_finished_requests() == {"req-a"} + + def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch): store = MagicMock() store.setup.return_value = 0 @@ -2172,6 +2220,39 @@ def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch): assert [db.block_size for db in store_worker.token_dbs] == [800, 800] +@pytest.mark.parametrize( + ("disable_hybrid_kv_cache_manager", "expected_is_hma_required"), + [ + (True, False), + (False, True), + ], +) +def test_worker_is_hma_required_from_kv_cache_groups( + monkeypatch, disable_hybrid_kv_cache_manager, expected_is_hma_required +): + store = MagicMock() + store.setup.return_value = 0 + _install_fake_mooncake(monkeypatch, store) + _patch_worker_runtime(monkeypatch) + monkeypatch.setattr( + worker.MooncakeStoreConfig, + "load_from_config", + staticmethod(lambda: _make_config()), + ) + vllm_config = _make_vllm_config( + disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager + ) + vllm_config.cache_config.block_size = 800 + vllm_config.cache_config.enable_prefix_caching = True + vllm_config.cache_config.prefix_match_unit = None + + store_worker = worker.MooncakeStoreWorker( + vllm_config, _make_qsa_hybrid_kv_cache_config() + ) + + assert store_worker._is_hma_required is expected_is_hma_required + + def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch): store = MagicMock() store.setup.return_value = 0 @@ -3345,6 +3426,7 @@ def _make_bare_worker( worker.can_put = kv_role in ("kv_producer", "kv_both") or save_decode_cache worker._capacity_only = False worker.block_size = block_size + worker._is_hma_required = False worker.tp_rank = 0 worker.enable_kv_events = False worker.load_async = True diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index 98633c233b..62e13fa438 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -155,7 +155,6 @@ def create_scheduler( vllm_config: VllmConfig, num_blocks: int = 10000, kv_cache_config: KVCacheConfig | None = None, - hash_block_size: int | None = None, ) -> Scheduler | AsyncScheduler: """Initialize Scheduler For Testing.""" block_size = vllm_config.cache_config.block_size @@ -186,7 +185,6 @@ def create_scheduler( log_stats=True, structured_output_manager=StructuredOutputManager(vllm_config), block_size=block_size, - hash_block_size=hash_block_size, ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py index 4451f0566d..5a3f462577 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py @@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorBase_V1, KVConnectorMetadata, KVConnectorRole, + KVConnectorTransferResults, SupportsHMA, ) from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats @@ -587,6 +588,13 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA): assert self.connector_worker is not None return self.connector_worker.get_finished() + def get_transfer_results( + self, finished_req_ids: set[str] + ) -> KVConnectorTransferResults: + """Get finished and failed sends/recvs from the worker.""" + assert self.connector_worker is not None + return self.connector_worker.get_transfer_results() + def get_block_ids_with_load_errors(self) -> set[int]: """Get block IDs whose remote KV load failed.""" assert self.connector_worker is not None @@ -1046,6 +1054,16 @@ class MooncakeConnectorWorker: self.finished_recving_reqs: set[ReqId] = set() # Written from the receiver loop, drained from the worker thread. self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue() + self._is_hma_required = ( + not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager + and any( + not isinstance(g.kv_cache_spec, FullAttentionSpec) + for g in kv_cache_config.transfer_groups + ) + ) + # Block IDs are only unique within a group; with HMA the scheduler + # tracks a single merged group, so failures are reported per request. + self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue() self.xfer_stats = MooncakeKVConnectorStats() @@ -1866,6 +1884,29 @@ class MooncakeConnectorWorker: return finished_sending_reqs or None, finished_recving_reqs or None + def get_transfer_results(self) -> KVConnectorTransferResults: + """ + Get transfers that completed on this specific worker, including + requests whose remote KV load failed. + + The scheduler process (via the MultiprocExecutor) will use this output + to track which workers are done. + """ + finished_sending_reqs, finished_recving_reqs = self.get_finished() + + failed_recving_reqs: set[ReqId] = set() + while True: + try: + failed_recving_reqs.add(self._failed_recv_reqs.get_nowait()) + except queue.Empty: + break + + return KVConnectorTransferResults( + finished_sending=set(finished_sending_reqs or ()), + finished_recving=set(finished_recving_reqs or ()), + failed_recving=failed_recving_reqs, + ) + def get_kv_connector_stats(self) -> KVConnectorStats | None: """Return transfer stats collected since the last call, or None if nothing has been recorded in this interval.""" @@ -1959,7 +2000,10 @@ class MooncakeConnectorWorker: # request is waiting on a load, and reporting one here would trip # the scheduler's `assert req_id in self.requests`. continue - self._invalid_block_ids.put(invalid) + if self._is_hma_required: + self._failed_recv_reqs.put(pull_meta.d_req_id) + else: + self._invalid_block_ids.put(invalid) self.finished_recving_reqs.add(pull_meta.d_req_id) if failed: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py index 134d5055b6..eb4a7fafe9 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py @@ -27,6 +27,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorBase_V1, KVConnectorMetadata, KVConnectorRole, + KVConnectorTransferResults, KVConnectorWorkerMetadata, SupportsHMA, ) @@ -359,6 +360,14 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): assert isinstance(metadata, MooncakeStoreConnectorMetadata) return self.connector_worker.get_finished(finished_req_ids, metadata) + def get_transfer_results( + self, finished_req_ids: set[str] + ) -> KVConnectorTransferResults: + assert self.connector_worker is not None + metadata = self._get_connector_metadata() + assert isinstance(metadata, MooncakeStoreConnectorMetadata) + return self.connector_worker.get_transfer_results(finished_req_ids, metadata) + def get_block_ids_with_load_errors(self) -> set[int]: assert self.connector_worker is not None return self.connector_worker.get_block_ids_with_load_errors() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 6f154ea991..0231ba7394 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -36,6 +36,9 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.distributed.kv_events import BlockStored +from vllm.distributed.kv_transfer.kv_connector.v1.base import ( + KVConnectorTransferResults, +) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501 ExternalCachedBlockPool, @@ -1265,6 +1268,7 @@ class KVCacheStoreRecvingThread(KVTransferThread): record_operation: Callable[..., None] | None = None, request_queue: queue.Queue[Any] | None = None, group_participates: Sequence[bool] | None = None, + is_hma_required: bool = False, ): super().__init__( store, @@ -1284,6 +1288,10 @@ class KVCacheStoreRecvingThread(KVTransferThread): # _invalid_block_ids can be access by both the Worker and RecvingThread self._invalid_block_ids_lock = threading.Lock() self._invalid_block_ids: set[int] = set() + # With HMA the scheduler tracks a single merged group while block IDs + # are only unique within a group, so failures are reported per request. + self._is_hma_required = is_hma_required + self._failed_requests: set[str] = set() self.disk_offload_buffer_budget_bytes = disk_offload_buffer_budget_bytes self.usable_disk_offload_buffer_budget_bytes = ( None @@ -1298,6 +1306,22 @@ class KVCacheStoreRecvingThread(KVTransferThread): with self._invalid_block_ids_lock: self._invalid_block_ids.update(block_ids) + def set_failed_request(self, req_id: str): + with self.done_task_lock: + self._failed_requests.add(req_id) + + def get_and_clear_failed_requests(self) -> set[str]: + with self.done_task_lock: + failed = self._failed_requests.copy() + self._failed_requests.clear() + return failed + + def _report_load_error(self, req_id: str, block_ids: list[int]) -> None: + if self._is_hma_required: + self.set_failed_request(req_id) + else: + self._add_load_error_block_ids(block_ids) + def get_and_clear_block_ids_with_load_errors(self) -> set[int]: with self._invalid_block_ids_lock: invalid_block_ids = self._invalid_block_ids.copy() @@ -1392,7 +1416,7 @@ class KVCacheStoreRecvingThread(KVTransferThread): # Mark every block: we skip the whole request, and the # tp_rank rotation means oversized_key isn't necessarily # the first block in the request's original order. - self._add_load_error_block_ids(block_id_list_c) + self._report_load_error(req_id, block_id_list_c) oversized_key_bytes = _estimate_disk_offload_staging_bytes( size_list_c[oversized_key_index] ) @@ -1465,8 +1489,8 @@ class KVCacheStoreRecvingThread(KVTransferThread): num_failed_keys=len(failed), ) if failed: - self._add_load_error_block_ids( - [block_id for _, _, block_id in failed] + self._report_load_error( + req_id, [block_id for _, _, block_id in failed] ) logger.warning( "Failed to get %d Mooncake keys from sub-batch " @@ -1477,7 +1501,7 @@ class KVCacheStoreRecvingThread(KVTransferThread): ) break except Exception as e: - self._add_load_error_block_ids(current_batch_block_ids) + self._report_load_error(req_id, current_batch_block_ids) self._record_operation( "load_get", load_get_start, @@ -1551,6 +1575,13 @@ class MooncakeStoreWorker: and not self.can_put ) self.cache_config = vllm_config.cache_config + self._is_hma_required = ( + not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager + and any( + not isinstance(g.kv_cache_spec, FullAttentionSpec) + for g in kv_cache_config.transfer_groups + ) + ) self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes( kv_cache_config, vllm_config ) @@ -2096,6 +2127,7 @@ class MooncakeStoreWorker: group.kv_cache_spec.prefix_cacheable for group in self._kv_cache_groups ], + is_hma_required=self._is_hma_required, ) recv_thread.name = f"KVCacheStoreRecvingThread-{i}" recv_thread.start() @@ -2185,6 +2217,26 @@ class MooncakeStoreWorker: block_ids |= recv_thread.get_and_clear_block_ids_with_load_errors() return block_ids + def get_transfer_results( + self, finished_req_ids: set[str], meta: MooncakeStoreConnectorMetadata + ) -> KVConnectorTransferResults: + """Get completed sends/recvs plus requests whose remote KV load failed.""" + done_sending, done_recving = self.get_finished(finished_req_ids, meta) + + if self._capacity_only: + return KVConnectorTransferResults(done_sending, done_recving) + + failed_recving: set[str] = set() + if self.load_async: + for recv_thread in self.kv_recv_threads: + failed_recving |= recv_thread.get_and_clear_failed_requests() + + return KVConnectorTransferResults( + finished_sending=done_sending, + finished_recving=done_recving, + failed_recving=failed_recving, + ) + def _record_kv_connector_operation( self, operation: str, diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index f1e11e9b1f..c1e956b9ec 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -3027,11 +3027,6 @@ class Scheduler(SchedulerInterface): For observability, it also accumulates the total number of tokens that will need to be recomputed across all affected requests. - A hybrid model spreads one token span over KV cache groups that disagree - on block size, so a block index maps to a different token position in - each group and there is no single longest valid prefix. Those requests - recompute in full instead. - Args: requests: The set of requests to scan for invalid blocks. invalid_block_ids: IDs of invalid blocks. @@ -3055,49 +3050,19 @@ class Scheduler(SchedulerInterface): # these requests must be rescheduled, but only the first will recompute # it. This set tracks blocks already marked for recomputation. marked_invalid_block_ids: set[int] = set() - # Positions a group holds no KV for -- skipped sliding-window blocks, - # and every Mamba block but the aligned snapshot -- all point at this - # one block, shared by every request. It carries no request's data, so - # it can neither fail for a request nor be evicted on its behalf. - null_block_id = self.kv_cache_manager.block_pool.null_block.block_id for request in requests: is_affected = False marked_invalid_block = False req_id = request.request_id - # One block-id list per KV cache group. - req_block_ids_per_group = self.kv_cache_manager.get_block_ids(req_id) + # One block-id list per KV cache group; single-group layouts are + # enforced by _handle_invalid_blocks. + (req_block_ids,) = self.kv_cache_manager.get_block_ids(req_id) # We iterate only over blocks that may contain externally computed # tokens req_num_computed_tokens = ( request.num_computed_tokens - num_scheduled_tokens.get(req_id, 0) ) - if len(req_block_ids_per_group) > 1: - # No single truncation point exists across groups, so the whole - # request is recomputed. Blocks are still marked, so a request - # sharing one of them is rescheduled too. - request_invalid_block_ids = { - block_id - for req_block_ids in req_block_ids_per_group - for block_id in req_block_ids - if block_id != null_block_id and block_id in invalid_block_ids - } - if request_invalid_block_ids: - marked_invalid_block_ids |= request_invalid_block_ids - total_affected_tokens += req_num_computed_tokens - request.num_computed_tokens = 0 - if evict_blocks: - for req_block_ids in req_block_ids_per_group: - blocks_to_evict.update( - block_id - for block_id in req_block_ids - if block_id != null_block_id - ) - affected_req_ids.add(req_id) - continue - - (req_block_ids,) = req_block_ids_per_group - req_num_computed_blocks = ( req_num_computed_tokens + self.block_size - 1 ) // self.block_size @@ -3175,6 +3140,17 @@ class Scheduler(SchedulerInterface): Returns: Set of affected request IDs to skip in update_from_output main loop. """ + if len(self.kv_cache_config.kv_cache_groups) > 1: + # Block IDs are only unique within a group, so a flat set of + # invalid block IDs cannot be mapped back to requests. + raise RuntimeError( + "A KV connector reported block-level load failures " + "(invalid_block_ids) on a layout with multiple KV cache " + "groups, where block IDs are only unique within a group. " + "Connectors must report failed requests via " + "KVConnectorTransferResults.failed_recving instead." + ) + should_fail = not self.recompute_kv_load_failures # handle async KV loads (not cached yet, evict_blocks=False)