[Bugfix][Mooncake] Report request-level KV load failures under HMA (#56855)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-09-16 14:55:40 +02:00
committed by GitHub
parent 903285fbcc
commit 8b1d188046
9 changed files with 329 additions and 377 deletions
@@ -2,21 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from typing import Literal
from unittest.mock import Mock, patch
from unittest.mock import Mock
import pytest
import torch
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
)
from vllm.v1.request import FinishReason, Request, RequestStatus
from vllm.v1.request import Request, RequestStatus
from .utils import (
create_model_runner_output,
@@ -44,18 +35,6 @@ def scheduler():
return create_scheduler(vllm_config)
@pytest.fixture
def hybrid_scheduler():
"""Scheduler for a hybrid model: full attention plus a Mamba group."""
vllm_config = create_vllm_config(kv_load_failure_policy="recompute")
kv_cache_config = make_kv_cache_config(
block_size=vllm_config.cache_config.block_size,
mamba_enabled=True,
num_blocks=10000,
)
return create_scheduler(vllm_config, kv_cache_config=kv_cache_config)
@pytest.mark.parametrize(
"num_prompt_blocks,num_external_computed_blocks,invalid_block_idxs",
[
@@ -361,317 +340,19 @@ def test_async_progressive_load_failure(
assert scheduler.connector.get_num_new_matched_tokens.call_count == 1
def _schedule_hybrid_async_load(
scheduler: Scheduler, num_prompt_blocks: int, num_external_computed_blocks: int
) -> Request:
"""Put one request into an async external load on a hybrid model."""
request = create_request(num_tokens=num_prompt_blocks * scheduler.block_size)
scheduler.add_request(request=request)
@pytest.mark.cpu_test
def test_block_level_load_failure_fails_closed_with_multiple_groups():
"""Block-level invalid-block reporting is rejected on multi-group layouts.
scheduler.connector = Mock()
scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(
{request.request_id: num_external_computed_blocks * scheduler.block_size},
async_load=True,
)
)
scheduler.connector.take_events.return_value = ()
return request
def _report_invalid_blocks(
scheduler: Scheduler, scheduler_output, invalid_block_ids: set[int]
) -> None:
scheduler.update_from_output(
scheduler_output,
create_model_runner_output(
reqs=[],
finished_recving=set(),
invalid_block_ids=invalid_block_ids,
use_eos=True,
),
)
@pytest.mark.parametrize("failed_group_idx", [0, 1])
def test_hybrid_load_failure_recomputes_whole_request(
hybrid_scheduler: Scheduler, failed_group_idx: int
):
"""A hybrid model must recover, not crash.
Its groups sit on different block grids, so there is no single truncation
point; the request restarts from scratch instead.
Block IDs are only unique within a group, so the scheduler cannot map
them back to requests; connectors must report failed_recving instead.
"""
num_prompt_blocks, num_external_computed_blocks = 10, 9
request = _schedule_hybrid_async_load(
hybrid_scheduler, num_prompt_blocks, num_external_computed_blocks
)
scheduler_output = hybrid_scheduler.schedule()
block_ids_per_group = hybrid_scheduler.kv_cache_manager.get_block_ids(
request.request_id
)
assert len(block_ids_per_group) == 2, "expected a hybrid (multi-group) layout"
null_block_id = hybrid_scheduler.kv_cache_manager.block_pool.null_block.block_id
# The Mamba group backs only its aligned snapshot with a real block, so
# pick a block that actually belongs to this request.
failed_block = next(
block_id
for block_id in block_ids_per_group[failed_group_idx]
if block_id != null_block_id
)
_report_invalid_blocks(hybrid_scheduler, scheduler_output, {failed_block})
assert request.num_computed_tokens == 0
assert hybrid_scheduler.failed_recving_kv_req_ids == {request.request_id}
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
def test_hybrid_load_failure_ignores_null_block(hybrid_scheduler: Scheduler):
"""The null block is shared by every request and holds no KV, so reporting
it must not restart anyone."""
num_prompt_blocks, num_external_computed_blocks = 10, 9
request = _schedule_hybrid_async_load(
hybrid_scheduler, num_prompt_blocks, num_external_computed_blocks
)
scheduler_output = hybrid_scheduler.schedule()
null_block_id = hybrid_scheduler.kv_cache_manager.block_pool.null_block.block_id
_, mamba_block_ids = hybrid_scheduler.kv_cache_manager.get_block_ids(
request.request_id
)
# The Mamba group parks its unaligned positions on the null block, so a
# request that never failed would otherwise be restarted by this report.
assert mamba_block_ids[0] == null_block_id
num_computed_tokens = request.num_computed_tokens
_report_invalid_blocks(hybrid_scheduler, scheduler_output, {null_block_id})
assert request.num_computed_tokens == num_computed_tokens
assert not hybrid_scheduler.failed_recving_kv_req_ids
def _make_deepseek_v4_five_group_config(num_blocks: int) -> KVCacheConfig:
"""Build the scheduler-side projection of V4-Flash's packed KV layout."""
def mla_spec() -> MLAAttentionSpec:
return MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=512,
dtype=torch.bfloat16,
tokens_per_state=4,
model_version="deepseek_v4",
)
def swa_spec(
*, block_size: int, sliding_window: int, head_size: int
) -> SlidingWindowMLASpec:
return SlidingWindowMLASpec(
block_size=block_size,
num_kv_heads=1,
head_size=head_size,
dtype=torch.float32,
sliding_window=sliding_window,
model_version="deepseek_v4",
)
# DeepSeek-V4 packs these into five physical groups. The two SWA groups
# have identical scheduling semantics but are split to align layer tuples.
groups = [
KVCacheGroupSpec(["full_mla_and_indexer"], mla_spec()),
KVCacheGroupSpec(
["c4_state"],
swa_spec(block_size=4, sliding_window=8, head_size=2048),
),
KVCacheGroupSpec(
["c128_state"],
swa_spec(block_size=8, sliding_window=128, head_size=2048),
),
KVCacheGroupSpec(
["swa.0"],
swa_spec(block_size=64, sliding_window=512, head_size=512),
),
KVCacheGroupSpec(
["swa.1"],
swa_spec(block_size=64, sliding_window=512, head_size=512),
),
]
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[],
kv_cache_groups=groups,
)
def _create_deepseek_v4_scheduler(
kv_load_failure_policy: Literal["recompute", "fail"] = "recompute",
) -> Scheduler:
num_blocks = 10000
vllm_config = create_vllm_config(
block_size=256,
max_num_batched_tokens=4096,
max_model_len=8192,
# max_model_len only reaches SchedulerConfig; opt-125m still caps the
# model at 2048, which is below the 9-block requests used here.
hf_overrides={"max_position_embeddings": 8192},
kv_load_failure_policy=kv_load_failure_policy,
)
return create_scheduler(
vllm_config = create_vllm_config()
scheduler = create_scheduler(
vllm_config,
num_blocks=num_blocks,
kv_cache_config=_make_deepseek_v4_five_group_config(num_blocks),
hash_block_size=4,
kv_cache_config=make_kv_cache_config(block_size=16, swa_enabled=True),
)
assert len(scheduler.kv_cache_config.kv_cache_groups) == 2
@pytest.fixture
def deepseek_v4_scheduler() -> Scheduler:
return _create_deepseek_v4_scheduler()
def _schedule_deepseek_v4_async_load(
scheduler: Scheduler,
) -> tuple[Request, SchedulerOutput]:
num_external_tokens = 8 * scheduler.block_size
request = create_request(num_tokens=9 * scheduler.block_size, block_size=4)
scheduler.add_request(request)
scheduler.connector = Mock()
scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(
{request.request_id: num_external_tokens}, async_load=True
)
)
scheduler.connector.take_events.return_value = ()
return request, scheduler.schedule()
@pytest.mark.parametrize("failed_group_idx", range(5))
def test_deepseek_v4_load_failure_recomputes_whole_request(
deepseek_v4_scheduler: Scheduler, failed_group_idx: int
):
"""A failed load in any V4-Flash KV group must not kill EngineCore."""
request, scheduler_output = _schedule_deepseek_v4_async_load(deepseek_v4_scheduler)
block_ids_per_group = deepseek_v4_scheduler.kv_cache_manager.get_block_ids(
request.request_id
)
assert len(block_ids_per_group) == 5
null_block_id = (
deepseek_v4_scheduler.kv_cache_manager.block_pool.null_block.block_id
)
failed_block_id = next(
block_id
for block_id in block_ids_per_group[failed_group_idx]
if block_id != null_block_id
)
deepseek_v4_scheduler.update_from_output(
scheduler_output,
create_model_runner_output(
reqs=[],
finished_recving=set(),
invalid_block_ids={failed_block_id},
use_eos=True,
),
)
assert request.num_computed_tokens == 0
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert deepseek_v4_scheduler.failed_recving_kv_req_ids == {request.request_id}
def test_deepseek_v4_load_failure_ignores_shared_null_block(
deepseek_v4_scheduler: Scheduler,
):
"""The shared null block contains no request KV and cannot fail a load."""
request, scheduler_output = _schedule_deepseek_v4_async_load(deepseek_v4_scheduler)
null_block_id = (
deepseek_v4_scheduler.kv_cache_manager.block_pool.null_block.block_id
)
block_ids_per_group = deepseek_v4_scheduler.kv_cache_manager.get_block_ids(
request.request_id
)
assert any(null_block_id in group for group in block_ids_per_group[1:])
num_computed_tokens = request.num_computed_tokens
deepseek_v4_scheduler.update_from_output(
scheduler_output,
create_model_runner_output(
reqs=[],
finished_recving=set(),
invalid_block_ids={null_block_id},
use_eos=True,
),
)
assert request.num_computed_tokens == num_computed_tokens
assert not deepseek_v4_scheduler.failed_recving_kv_req_ids
def test_deepseek_v4_sync_load_failure_fails_and_evicts_real_blocks():
"""Fail policy must terminate cleanly and never evict the null block."""
scheduler = _create_deepseek_v4_scheduler(kv_load_failure_policy="fail")
num_external_tokens = 8 * scheduler.block_size
request = create_request(num_tokens=9 * scheduler.block_size, block_size=4)
scheduler.add_request(request)
scheduler.connector = Mock()
scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(
{request.request_id: num_external_tokens}, async_load=False
)
)
scheduler.connector.request_finished.return_value = (False, None)
scheduler.connector.take_events.return_value = ()
scheduler_output = scheduler.schedule()
block_ids_per_group = scheduler.kv_cache_manager.get_block_ids(request.request_id)
assert len(block_ids_per_group) == 5
null_block_id = scheduler.kv_cache_manager.block_pool.null_block.block_id
real_block_ids = {
block_id
for group_block_ids in block_ids_per_group
for block_id in group_block_ids
if block_id != null_block_id
}
failed_block_id = next(
block_id for block_id in block_ids_per_group[2] if block_id != null_block_id
)
evicted_block_ids: list[set[int]] = []
original_evict_blocks = scheduler.kv_cache_manager.evict_blocks
def evict_blocks_spy(block_ids: set[int]) -> None:
evicted_block_ids.append(set(block_ids))
original_evict_blocks(block_ids)
with (
patch.object(
scheduler.kv_cache_manager,
"evict_blocks",
side_effect=evict_blocks_spy,
),
patch.object(scheduler, "_connector_finished", return_value=(False, None)),
):
outputs = scheduler.update_from_output(
scheduler_output,
create_model_runner_output(
[request],
invalid_block_ids={failed_block_id},
use_eos=True,
),
)
assert request.status == RequestStatus.FINISHED_ERROR
assert request.get_finished_reason() == FinishReason.ERROR
assert request.request_id not in scheduler.requests
assert evicted_block_ids == [real_block_ids]
assert null_block_id not in evicted_block_ids[0]
engine_outputs = next(iter(outputs.values()))
assert len(engine_outputs.outputs) == 1
assert engine_outputs.outputs[0].request_id == request.request_id
assert engine_outputs.outputs[0].finish_reason == FinishReason.ERROR
with pytest.raises(RuntimeError, match="failed_recving"):
scheduler._handle_invalid_blocks({1}, {})
@@ -18,6 +18,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im
MooncakeConnectorMetadata,
MooncakeConnectorScheduler,
MooncakeXferMetadata,
PullReqMeta,
SendBlockMeta,
TransferRegion,
)
@@ -436,3 +437,79 @@ def test_request_finished_with_hma_groups():
assert stored_blocks[0] == fa_blocks
# SW: clipped to last 9 blocks (sw_size=128, block_size=16 → 8+1=9)
assert stored_blocks[1] == sw_blocks[-9:]
# ---------------------------------------------------------------------------
# Worker-side load-failure reporting (HMA vs non-HMA)
# ---------------------------------------------------------------------------
def _make_kv_consumer_worker(swa_enabled: bool, disable_hma: bool = False):
block_size = 16
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_consumer",
block_size=block_size,
)
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma
kv_cache_config = make_kv_cache_config(
block_size=block_size, swa_enabled=swa_enabled
)
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
connector = MooncakeConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
return connector.connector_worker
def _make_pull_meta(d_req_id: str, local_block_ids: list[list[int]]) -> PullReqMeta:
return PullReqMeta(
d_req_id=d_req_id,
transfer_id="tx-1",
local_block_ids=local_block_ids,
remote_engine_id="engine-0",
remote_bootstrap_addr="127.0.0.1:1234",
)
# The tests below construct a MooncakeConnectorWorker, whose __init__
# requires an accelerator device, so (like the worker tests in
# test_mooncake_connector.py) they run on the GPU KV-connectors job only.
@pytest.mark.parametrize(
"swa_enabled,disable_hma,expected_is_hma",
[
(True, False, True), # SWA group present, HMA enabled
(True, True, False), # SWA group present, but HMA disabled
(False, False, False), # FA only, HMA not needed
],
)
def test_worker_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
"""Worker-side _is_hma_required mirrors the scheduler's derivation."""
worker = _make_kv_consumer_worker(swa_enabled, disable_hma)
assert worker._is_hma_required is expected_is_hma
def test_worker_failed_recv_reports_request_level_failure_with_hma():
"""With HMA, load failures report the request, not ambiguous block IDs."""
worker = _make_kv_consumer_worker(swa_enabled=True)
assert worker._is_hma_required
pull_meta = _make_pull_meta("d-req-1", [[1, 2], [3, 4]])
worker._handle_failed_recv({"p-req-1": pull_meta}, ["p-req-1"], "boom")
assert worker.get_block_ids_with_load_errors() == set()
results = worker.get_transfer_results()
assert results.failed_recving == {"d-req-1"}
assert results.finished_recving == {"d-req-1"}
def test_worker_failed_recv_reports_block_ids_without_hma():
"""Without HMA, load failures keep reporting block-level errors."""
worker = _make_kv_consumer_worker(swa_enabled=False)
assert not worker._is_hma_required
pull_meta = _make_pull_meta("d-req-1", [[1, 2], [3, 4]])
worker._handle_failed_recv({"p-req-1": pull_meta}, ["p-req-1"], "boom")
assert worker.get_block_ids_with_load_errors() == {1, 2, 3, 4}
results = worker.get_transfer_results()
assert results.failed_recving == set()
assert results.finished_recving == {"d-req-1"}
@@ -12,6 +12,7 @@ from vllm.config import set_current_vllm_config
from vllm.distributed.kv_events import BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorRole,
KVConnectorTransferResults,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
connector as mooncake_store_connector,
@@ -245,6 +246,38 @@ def test_worker_methods_delegate_to_store_worker():
assert invalid_block_ids == {3, 4}
def test_get_transfer_results_delegates_to_store_worker():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
metadata = MooncakeStoreConnectorMetadata(set(), set())
finished_req_ids = {"req-1"}
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"connector.MooncakeStoreWorker"
) as mock_worker_cls,
):
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
worker = mock_worker_cls.return_value
transfer_results = KVConnectorTransferResults(
finished_sending={"req-1"},
finished_recving={"req-2"},
failed_recving={"req-3"},
)
worker.get_transfer_results.return_value = transfer_results
connector.bind_connector_metadata(metadata)
result = connector.get_transfer_results(finished_req_ids)
worker.get_transfer_results.assert_called_once_with(finished_req_ids, metadata)
assert result is transfer_results
def test_get_kv_connector_kv_cache_events_returns_none_when_disabled():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
@@ -177,6 +177,7 @@ def _make_store_recving_thread(
*,
tp_rank: int = 0,
disk_offload_buffer_budget_bytes: int | None = None,
is_hma_required: bool = False,
) -> mooncake_store_worker.KVCacheStoreRecvingThread:
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
@@ -199,6 +200,7 @@ def _make_store_recving_thread(
ready_event=threading.Event(),
coord=coord,
disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes,
is_hma_required=is_hma_required,
)
thread.request_queue.task_done = MagicMock()
return thread
@@ -303,6 +305,7 @@ def _make_vllm_config(
kv_role: str = "kv_both",
pipeline_parallel_size: int = 1,
kv_cache_layout: KVCacheLayout = KVCacheLayout.LBHNC,
disable_hybrid_kv_cache_manager: bool = True,
) -> SimpleNamespace:
cache_config = SimpleNamespace(block_size=16, num_gpu_blocks=10)
cache_config.get_resolved_kv_cache_layout = lambda: kv_cache_layout
@@ -319,6 +322,9 @@ def _make_vllm_config(
kv_role=kv_role, extra_config=extra_config
),
cache_config=cache_config,
scheduler_config=SimpleNamespace(
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager
),
kv_events_config=SimpleNamespace(enable_kv_cache_events=False),
speculative_config=None,
)
@@ -2147,6 +2153,48 @@ def test_recv_thread_reports_unsplittable_key_larger_than_budget():
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2}
def test_recv_thread_partial_load_failure_reports_block_ids_without_hma():
store = MagicMock()
# First key fails, second succeeds.
store.batch_get_into_multi_buffers.return_value = [-1, 0]
thread = _make_store_recving_thread(store)
thread._handle_request(_make_load_req("req-a", [b"h0", b"h1"], token_len=32))
assert thread.get_and_clear_block_ids_with_load_errors() == {0}
assert thread.get_and_clear_failed_requests() == set()
assert thread.get_and_clear_finished_requests() == {"req-a"}
def test_recv_thread_partial_load_failure_reports_request_with_hma():
store = MagicMock()
store.batch_get_into_multi_buffers.return_value = [-1, 0]
thread = _make_store_recving_thread(store, is_hma_required=True)
thread._handle_request(_make_load_req("req-a", [b"h0", b"h1"], token_len=32))
assert thread.get_and_clear_block_ids_with_load_errors() == set()
assert thread.get_and_clear_failed_requests() == {"req-a"}
assert thread.get_and_clear_finished_requests() == {"req-a"}
def test_recv_thread_oversized_key_reports_request_with_hma():
store = MagicMock()
thread = _make_store_recving_thread(
store,
tp_rank=2,
disk_offload_buffer_budget_bytes=_DISK_OFFLOAD_BUDGET_TOO_SMALL,
is_hma_required=True,
)
thread._handle_request(_make_load_req("req-a", [b"a0", b"a1", b"a2"], token_len=48))
assert store.batch_get_into_multi_buffers.call_count == 0
assert thread.get_and_clear_block_ids_with_load_errors() == set()
assert thread.get_and_clear_failed_requests() == {"req-a"}
assert thread.get_and_clear_finished_requests() == {"req-a"}
def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch):
store = MagicMock()
store.setup.return_value = 0
@@ -2172,6 +2220,39 @@ def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch):
assert [db.block_size for db in store_worker.token_dbs] == [800, 800]
@pytest.mark.parametrize(
("disable_hybrid_kv_cache_manager", "expected_is_hma_required"),
[
(True, False),
(False, True),
],
)
def test_worker_is_hma_required_from_kv_cache_groups(
monkeypatch, disable_hybrid_kv_cache_manager, expected_is_hma_required
):
store = MagicMock()
store.setup.return_value = 0
_install_fake_mooncake(monkeypatch, store)
_patch_worker_runtime(monkeypatch)
monkeypatch.setattr(
worker.MooncakeStoreConfig,
"load_from_config",
staticmethod(lambda: _make_config()),
)
vllm_config = _make_vllm_config(
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager
)
vllm_config.cache_config.block_size = 800
vllm_config.cache_config.enable_prefix_caching = True
vllm_config.cache_config.prefix_match_unit = None
store_worker = worker.MooncakeStoreWorker(
vllm_config, _make_qsa_hybrid_kv_cache_config()
)
assert store_worker._is_hma_required is expected_is_hma_required
def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
store = MagicMock()
store.setup.return_value = 0
@@ -3345,6 +3426,7 @@ def _make_bare_worker(
worker.can_put = kv_role in ("kv_producer", "kv_both") or save_decode_cache
worker._capacity_only = False
worker.block_size = block_size
worker._is_hma_required = False
worker.tp_rank = 0
worker.enable_kv_events = False
worker.load_async = True
-2
View File
@@ -155,7 +155,6 @@ def create_scheduler(
vllm_config: VllmConfig,
num_blocks: int = 10000,
kv_cache_config: KVCacheConfig | None = None,
hash_block_size: int | None = None,
) -> Scheduler | AsyncScheduler:
"""Initialize Scheduler For Testing."""
block_size = vllm_config.cache_config.block_size
@@ -186,7 +185,6 @@ def create_scheduler(
log_stats=True,
structured_output_manager=StructuredOutputManager(vllm_config),
block_size=block_size,
hash_block_size=hash_block_size,
)