[Bugfix][Mooncake] Report request-level KV load failures under HMA (#56855)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-09-16 14:55:40 +02:00
committed by GitHub
parent 903285fbcc
commit 8b1d188046
9 changed files with 329 additions and 377 deletions
@@ -2,21 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from typing import Literal
from unittest.mock import Mock, patch
from unittest.mock import Mock
import pytest
import torch
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
)
from vllm.v1.request import FinishReason, Request, RequestStatus
from vllm.v1.request import Request, RequestStatus
from .utils import (
create_model_runner_output,
@@ -44,18 +35,6 @@ def scheduler():
return create_scheduler(vllm_config)
@pytest.fixture
def hybrid_scheduler():
"""Scheduler for a hybrid model: full attention plus a Mamba group."""
vllm_config = create_vllm_config(kv_load_failure_policy="recompute")
kv_cache_config = make_kv_cache_config(
block_size=vllm_config.cache_config.block_size,
mamba_enabled=True,
num_blocks=10000,
)
return create_scheduler(vllm_config, kv_cache_config=kv_cache_config)
@pytest.mark.parametrize(
"num_prompt_blocks,num_external_computed_blocks,invalid_block_idxs",
[
@@ -361,317 +340,19 @@ def test_async_progressive_load_failure(
assert scheduler.connector.get_num_new_matched_tokens.call_count == 1
def _schedule_hybrid_async_load(
scheduler: Scheduler, num_prompt_blocks: int, num_external_computed_blocks: int
) -> Request:
"""Put one request into an async external load on a hybrid model."""
request = create_request(num_tokens=num_prompt_blocks * scheduler.block_size)
scheduler.add_request(request=request)
@pytest.mark.cpu_test
def test_block_level_load_failure_fails_closed_with_multiple_groups():
"""Block-level invalid-block reporting is rejected on multi-group layouts.
scheduler.connector = Mock()
scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(
{request.request_id: num_external_computed_blocks * scheduler.block_size},
async_load=True,
)
)
scheduler.connector.take_events.return_value = ()
return request
def _report_invalid_blocks(
scheduler: Scheduler, scheduler_output, invalid_block_ids: set[int]
) -> None:
scheduler.update_from_output(
scheduler_output,
create_model_runner_output(
reqs=[],
finished_recving=set(),
invalid_block_ids=invalid_block_ids,
use_eos=True,
),
)
@pytest.mark.parametrize("failed_group_idx", [0, 1])
def test_hybrid_load_failure_recomputes_whole_request(
hybrid_scheduler: Scheduler, failed_group_idx: int
):
"""A hybrid model must recover, not crash.
Its groups sit on different block grids, so there is no single truncation
point; the request restarts from scratch instead.
Block IDs are only unique within a group, so the scheduler cannot map
them back to requests; connectors must report failed_recving instead.
"""
num_prompt_blocks, num_external_computed_blocks = 10, 9
request = _schedule_hybrid_async_load(
hybrid_scheduler, num_prompt_blocks, num_external_computed_blocks
)
scheduler_output = hybrid_scheduler.schedule()
block_ids_per_group = hybrid_scheduler.kv_cache_manager.get_block_ids(
request.request_id
)
assert len(block_ids_per_group) == 2, "expected a hybrid (multi-group) layout"
null_block_id = hybrid_scheduler.kv_cache_manager.block_pool.null_block.block_id
# The Mamba group backs only its aligned snapshot with a real block, so
# pick a block that actually belongs to this request.
failed_block = next(
block_id
for block_id in block_ids_per_group[failed_group_idx]
if block_id != null_block_id
)
_report_invalid_blocks(hybrid_scheduler, scheduler_output, {failed_block})
assert request.num_computed_tokens == 0
assert hybrid_scheduler.failed_recving_kv_req_ids == {request.request_id}
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
def test_hybrid_load_failure_ignores_null_block(hybrid_scheduler: Scheduler):
"""The null block is shared by every request and holds no KV, so reporting
it must not restart anyone."""
num_prompt_blocks, num_external_computed_blocks = 10, 9
request = _schedule_hybrid_async_load(
hybrid_scheduler, num_prompt_blocks, num_external_computed_blocks
)
scheduler_output = hybrid_scheduler.schedule()
null_block_id = hybrid_scheduler.kv_cache_manager.block_pool.null_block.block_id
_, mamba_block_ids = hybrid_scheduler.kv_cache_manager.get_block_ids(
request.request_id
)
# The Mamba group parks its unaligned positions on the null block, so a
# request that never failed would otherwise be restarted by this report.
assert mamba_block_ids[0] == null_block_id
num_computed_tokens = request.num_computed_tokens
_report_invalid_blocks(hybrid_scheduler, scheduler_output, {null_block_id})
assert request.num_computed_tokens == num_computed_tokens
assert not hybrid_scheduler.failed_recving_kv_req_ids
def _make_deepseek_v4_five_group_config(num_blocks: int) -> KVCacheConfig:
"""Build the scheduler-side projection of V4-Flash's packed KV layout."""
def mla_spec() -> MLAAttentionSpec:
return MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=512,
dtype=torch.bfloat16,
tokens_per_state=4,
model_version="deepseek_v4",
)
def swa_spec(
*, block_size: int, sliding_window: int, head_size: int
) -> SlidingWindowMLASpec:
return SlidingWindowMLASpec(
block_size=block_size,
num_kv_heads=1,
head_size=head_size,
dtype=torch.float32,
sliding_window=sliding_window,
model_version="deepseek_v4",
)
# DeepSeek-V4 packs these into five physical groups. The two SWA groups
# have identical scheduling semantics but are split to align layer tuples.
groups = [
KVCacheGroupSpec(["full_mla_and_indexer"], mla_spec()),
KVCacheGroupSpec(
["c4_state"],
swa_spec(block_size=4, sliding_window=8, head_size=2048),
),
KVCacheGroupSpec(
["c128_state"],
swa_spec(block_size=8, sliding_window=128, head_size=2048),
),
KVCacheGroupSpec(
["swa.0"],
swa_spec(block_size=64, sliding_window=512, head_size=512),
),
KVCacheGroupSpec(
["swa.1"],
swa_spec(block_size=64, sliding_window=512, head_size=512),
),
]
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[],
kv_cache_groups=groups,
)
def _create_deepseek_v4_scheduler(
kv_load_failure_policy: Literal["recompute", "fail"] = "recompute",
) -> Scheduler:
num_blocks = 10000
vllm_config = create_vllm_config(
block_size=256,
max_num_batched_tokens=4096,
max_model_len=8192,
# max_model_len only reaches SchedulerConfig; opt-125m still caps the
# model at 2048, which is below the 9-block requests used here.
hf_overrides={"max_position_embeddings": 8192},
kv_load_failure_policy=kv_load_failure_policy,
)
return create_scheduler(
vllm_config = create_vllm_config()
scheduler = create_scheduler(
vllm_config,
num_blocks=num_blocks,
kv_cache_config=_make_deepseek_v4_five_group_config(num_blocks),
hash_block_size=4,
kv_cache_config=make_kv_cache_config(block_size=16, swa_enabled=True),
)
assert len(scheduler.kv_cache_config.kv_cache_groups) == 2
@pytest.fixture
def deepseek_v4_scheduler() -> Scheduler:
return _create_deepseek_v4_scheduler()
def _schedule_deepseek_v4_async_load(
scheduler: Scheduler,
) -> tuple[Request, SchedulerOutput]:
num_external_tokens = 8 * scheduler.block_size
request = create_request(num_tokens=9 * scheduler.block_size, block_size=4)
scheduler.add_request(request)
scheduler.connector = Mock()
scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(
{request.request_id: num_external_tokens}, async_load=True
)
)
scheduler.connector.take_events.return_value = ()
return request, scheduler.schedule()
@pytest.mark.parametrize("failed_group_idx", range(5))
def test_deepseek_v4_load_failure_recomputes_whole_request(
deepseek_v4_scheduler: Scheduler, failed_group_idx: int
):
"""A failed load in any V4-Flash KV group must not kill EngineCore."""
request, scheduler_output = _schedule_deepseek_v4_async_load(deepseek_v4_scheduler)
block_ids_per_group = deepseek_v4_scheduler.kv_cache_manager.get_block_ids(
request.request_id
)
assert len(block_ids_per_group) == 5
null_block_id = (
deepseek_v4_scheduler.kv_cache_manager.block_pool.null_block.block_id
)
failed_block_id = next(
block_id
for block_id in block_ids_per_group[failed_group_idx]
if block_id != null_block_id
)
deepseek_v4_scheduler.update_from_output(
scheduler_output,
create_model_runner_output(
reqs=[],
finished_recving=set(),
invalid_block_ids={failed_block_id},
use_eos=True,
),
)
assert request.num_computed_tokens == 0
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert deepseek_v4_scheduler.failed_recving_kv_req_ids == {request.request_id}
def test_deepseek_v4_load_failure_ignores_shared_null_block(
deepseek_v4_scheduler: Scheduler,
):
"""The shared null block contains no request KV and cannot fail a load."""
request, scheduler_output = _schedule_deepseek_v4_async_load(deepseek_v4_scheduler)
null_block_id = (
deepseek_v4_scheduler.kv_cache_manager.block_pool.null_block.block_id
)
block_ids_per_group = deepseek_v4_scheduler.kv_cache_manager.get_block_ids(
request.request_id
)
assert any(null_block_id in group for group in block_ids_per_group[1:])
num_computed_tokens = request.num_computed_tokens
deepseek_v4_scheduler.update_from_output(
scheduler_output,
create_model_runner_output(
reqs=[],
finished_recving=set(),
invalid_block_ids={null_block_id},
use_eos=True,
),
)
assert request.num_computed_tokens == num_computed_tokens
assert not deepseek_v4_scheduler.failed_recving_kv_req_ids
def test_deepseek_v4_sync_load_failure_fails_and_evicts_real_blocks():
"""Fail policy must terminate cleanly and never evict the null block."""
scheduler = _create_deepseek_v4_scheduler(kv_load_failure_policy="fail")
num_external_tokens = 8 * scheduler.block_size
request = create_request(num_tokens=9 * scheduler.block_size, block_size=4)
scheduler.add_request(request)
scheduler.connector = Mock()
scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(
{request.request_id: num_external_tokens}, async_load=False
)
)
scheduler.connector.request_finished.return_value = (False, None)
scheduler.connector.take_events.return_value = ()
scheduler_output = scheduler.schedule()
block_ids_per_group = scheduler.kv_cache_manager.get_block_ids(request.request_id)
assert len(block_ids_per_group) == 5
null_block_id = scheduler.kv_cache_manager.block_pool.null_block.block_id
real_block_ids = {
block_id
for group_block_ids in block_ids_per_group
for block_id in group_block_ids
if block_id != null_block_id
}
failed_block_id = next(
block_id for block_id in block_ids_per_group[2] if block_id != null_block_id
)
evicted_block_ids: list[set[int]] = []
original_evict_blocks = scheduler.kv_cache_manager.evict_blocks
def evict_blocks_spy(block_ids: set[int]) -> None:
evicted_block_ids.append(set(block_ids))
original_evict_blocks(block_ids)
with (
patch.object(
scheduler.kv_cache_manager,
"evict_blocks",
side_effect=evict_blocks_spy,
),
patch.object(scheduler, "_connector_finished", return_value=(False, None)),
):
outputs = scheduler.update_from_output(
scheduler_output,
create_model_runner_output(
[request],
invalid_block_ids={failed_block_id},
use_eos=True,
),
)
assert request.status == RequestStatus.FINISHED_ERROR
assert request.get_finished_reason() == FinishReason.ERROR
assert request.request_id not in scheduler.requests
assert evicted_block_ids == [real_block_ids]
assert null_block_id not in evicted_block_ids[0]
engine_outputs = next(iter(outputs.values()))
assert len(engine_outputs.outputs) == 1
assert engine_outputs.outputs[0].request_id == request.request_id
assert engine_outputs.outputs[0].finish_reason == FinishReason.ERROR
with pytest.raises(RuntimeError, match="failed_recving"):
scheduler._handle_invalid_blocks({1}, {})
@@ -18,6 +18,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im
MooncakeConnectorMetadata,
MooncakeConnectorScheduler,
MooncakeXferMetadata,
PullReqMeta,
SendBlockMeta,
TransferRegion,
)
@@ -436,3 +437,79 @@ def test_request_finished_with_hma_groups():
assert stored_blocks[0] == fa_blocks
# SW: clipped to last 9 blocks (sw_size=128, block_size=16 → 8+1=9)
assert stored_blocks[1] == sw_blocks[-9:]
# ---------------------------------------------------------------------------
# Worker-side load-failure reporting (HMA vs non-HMA)
# ---------------------------------------------------------------------------
def _make_kv_consumer_worker(swa_enabled: bool, disable_hma: bool = False):
block_size = 16
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_consumer",
block_size=block_size,
)
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma
kv_cache_config = make_kv_cache_config(
block_size=block_size, swa_enabled=swa_enabled
)
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
connector = MooncakeConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
return connector.connector_worker
def _make_pull_meta(d_req_id: str, local_block_ids: list[list[int]]) -> PullReqMeta:
return PullReqMeta(
d_req_id=d_req_id,
transfer_id="tx-1",
local_block_ids=local_block_ids,
remote_engine_id="engine-0",
remote_bootstrap_addr="127.0.0.1:1234",
)
# The tests below construct a MooncakeConnectorWorker, whose __init__
# requires an accelerator device, so (like the worker tests in
# test_mooncake_connector.py) they run on the GPU KV-connectors job only.
@pytest.mark.parametrize(
"swa_enabled,disable_hma,expected_is_hma",
[
(True, False, True), # SWA group present, HMA enabled
(True, True, False), # SWA group present, but HMA disabled
(False, False, False), # FA only, HMA not needed
],
)
def test_worker_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
"""Worker-side _is_hma_required mirrors the scheduler's derivation."""
worker = _make_kv_consumer_worker(swa_enabled, disable_hma)
assert worker._is_hma_required is expected_is_hma
def test_worker_failed_recv_reports_request_level_failure_with_hma():
"""With HMA, load failures report the request, not ambiguous block IDs."""
worker = _make_kv_consumer_worker(swa_enabled=True)
assert worker._is_hma_required
pull_meta = _make_pull_meta("d-req-1", [[1, 2], [3, 4]])
worker._handle_failed_recv({"p-req-1": pull_meta}, ["p-req-1"], "boom")
assert worker.get_block_ids_with_load_errors() == set()
results = worker.get_transfer_results()
assert results.failed_recving == {"d-req-1"}
assert results.finished_recving == {"d-req-1"}
def test_worker_failed_recv_reports_block_ids_without_hma():
"""Without HMA, load failures keep reporting block-level errors."""
worker = _make_kv_consumer_worker(swa_enabled=False)
assert not worker._is_hma_required
pull_meta = _make_pull_meta("d-req-1", [[1, 2], [3, 4]])
worker._handle_failed_recv({"p-req-1": pull_meta}, ["p-req-1"], "boom")
assert worker.get_block_ids_with_load_errors() == {1, 2, 3, 4}
results = worker.get_transfer_results()
assert results.failed_recving == set()
assert results.finished_recving == {"d-req-1"}
@@ -12,6 +12,7 @@ from vllm.config import set_current_vllm_config
from vllm.distributed.kv_events import BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorRole,
KVConnectorTransferResults,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
connector as mooncake_store_connector,
@@ -245,6 +246,38 @@ def test_worker_methods_delegate_to_store_worker():
assert invalid_block_ids == {3, 4}
def test_get_transfer_results_delegates_to_store_worker():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
metadata = MooncakeStoreConnectorMetadata(set(), set())
finished_req_ids = {"req-1"}
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"connector.MooncakeStoreWorker"
) as mock_worker_cls,
):
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
)
worker = mock_worker_cls.return_value
transfer_results = KVConnectorTransferResults(
finished_sending={"req-1"},
finished_recving={"req-2"},
failed_recving={"req-3"},
)
worker.get_transfer_results.return_value = transfer_results
connector.bind_connector_metadata(metadata)
result = connector.get_transfer_results(finished_req_ids)
worker.get_transfer_results.assert_called_once_with(finished_req_ids, metadata)
assert result is transfer_results
def test_get_kv_connector_kv_cache_events_returns_none_when_disabled():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
@@ -177,6 +177,7 @@ def _make_store_recving_thread(
*,
tp_rank: int = 0,
disk_offload_buffer_budget_bytes: int | None = None,
is_hma_required: bool = False,
) -> mooncake_store_worker.KVCacheStoreRecvingThread:
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
@@ -199,6 +200,7 @@ def _make_store_recving_thread(
ready_event=threading.Event(),
coord=coord,
disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes,
is_hma_required=is_hma_required,
)
thread.request_queue.task_done = MagicMock()
return thread
@@ -303,6 +305,7 @@ def _make_vllm_config(
kv_role: str = "kv_both",
pipeline_parallel_size: int = 1,
kv_cache_layout: KVCacheLayout = KVCacheLayout.LBHNC,
disable_hybrid_kv_cache_manager: bool = True,
) -> SimpleNamespace:
cache_config = SimpleNamespace(block_size=16, num_gpu_blocks=10)
cache_config.get_resolved_kv_cache_layout = lambda: kv_cache_layout
@@ -319,6 +322,9 @@ def _make_vllm_config(
kv_role=kv_role, extra_config=extra_config
),
cache_config=cache_config,
scheduler_config=SimpleNamespace(
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager
),
kv_events_config=SimpleNamespace(enable_kv_cache_events=False),
speculative_config=None,
)
@@ -2147,6 +2153,48 @@ def test_recv_thread_reports_unsplittable_key_larger_than_budget():
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2}
def test_recv_thread_partial_load_failure_reports_block_ids_without_hma():
store = MagicMock()
# First key fails, second succeeds.
store.batch_get_into_multi_buffers.return_value = [-1, 0]
thread = _make_store_recving_thread(store)
thread._handle_request(_make_load_req("req-a", [b"h0", b"h1"], token_len=32))
assert thread.get_and_clear_block_ids_with_load_errors() == {0}
assert thread.get_and_clear_failed_requests() == set()
assert thread.get_and_clear_finished_requests() == {"req-a"}
def test_recv_thread_partial_load_failure_reports_request_with_hma():
store = MagicMock()
store.batch_get_into_multi_buffers.return_value = [-1, 0]
thread = _make_store_recving_thread(store, is_hma_required=True)
thread._handle_request(_make_load_req("req-a", [b"h0", b"h1"], token_len=32))
assert thread.get_and_clear_block_ids_with_load_errors() == set()
assert thread.get_and_clear_failed_requests() == {"req-a"}
assert thread.get_and_clear_finished_requests() == {"req-a"}
def test_recv_thread_oversized_key_reports_request_with_hma():
store = MagicMock()
thread = _make_store_recving_thread(
store,
tp_rank=2,
disk_offload_buffer_budget_bytes=_DISK_OFFLOAD_BUDGET_TOO_SMALL,
is_hma_required=True,
)
thread._handle_request(_make_load_req("req-a", [b"a0", b"a1", b"a2"], token_len=48))
assert store.batch_get_into_multi_buffers.call_count == 0
assert thread.get_and_clear_block_ids_with_load_errors() == set()
assert thread.get_and_clear_failed_requests() == {"req-a"}
assert thread.get_and_clear_finished_requests() == {"req-a"}
def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch):
store = MagicMock()
store.setup.return_value = 0
@@ -2172,6 +2220,39 @@ def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch):
assert [db.block_size for db in store_worker.token_dbs] == [800, 800]
@pytest.mark.parametrize(
("disable_hybrid_kv_cache_manager", "expected_is_hma_required"),
[
(True, False),
(False, True),
],
)
def test_worker_is_hma_required_from_kv_cache_groups(
monkeypatch, disable_hybrid_kv_cache_manager, expected_is_hma_required
):
store = MagicMock()
store.setup.return_value = 0
_install_fake_mooncake(monkeypatch, store)
_patch_worker_runtime(monkeypatch)
monkeypatch.setattr(
worker.MooncakeStoreConfig,
"load_from_config",
staticmethod(lambda: _make_config()),
)
vllm_config = _make_vllm_config(
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager
)
vllm_config.cache_config.block_size = 800
vllm_config.cache_config.enable_prefix_caching = True
vllm_config.cache_config.prefix_match_unit = None
store_worker = worker.MooncakeStoreWorker(
vllm_config, _make_qsa_hybrid_kv_cache_config()
)
assert store_worker._is_hma_required is expected_is_hma_required
def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
store = MagicMock()
store.setup.return_value = 0
@@ -3345,6 +3426,7 @@ def _make_bare_worker(
worker.can_put = kv_role in ("kv_producer", "kv_both") or save_decode_cache
worker._capacity_only = False
worker.block_size = block_size
worker._is_hma_required = False
worker.tp_rank = 0
worker.enable_kv_events = False
worker.load_async = True
-2
View File
@@ -155,7 +155,6 @@ def create_scheduler(
vllm_config: VllmConfig,
num_blocks: int = 10000,
kv_cache_config: KVCacheConfig | None = None,
hash_block_size: int | None = None,
) -> Scheduler | AsyncScheduler:
"""Initialize Scheduler For Testing."""
block_size = vllm_config.cache_config.block_size
@@ -186,7 +185,6 @@ def create_scheduler(
log_stats=True,
structured_output_manager=StructuredOutputManager(vllm_config),
block_size=block_size,
hash_block_size=hash_block_size,
)
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
@@ -587,6 +588,13 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
assert self.connector_worker is not None
return self.connector_worker.get_finished()
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
"""Get finished and failed sends/recvs from the worker."""
assert self.connector_worker is not None
return self.connector_worker.get_transfer_results()
def get_block_ids_with_load_errors(self) -> set[int]:
"""Get block IDs whose remote KV load failed."""
assert self.connector_worker is not None
@@ -1046,6 +1054,16 @@ class MooncakeConnectorWorker:
self.finished_recving_reqs: set[ReqId] = set()
# Written from the receiver loop, drained from the worker thread.
self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue()
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.transfer_groups
)
)
# Block IDs are only unique within a group; with HMA the scheduler
# tracks a single merged group, so failures are reported per request.
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
self.xfer_stats = MooncakeKVConnectorStats()
@@ -1866,6 +1884,29 @@ class MooncakeConnectorWorker:
return finished_sending_reqs or None, finished_recving_reqs or None
def get_transfer_results(self) -> KVConnectorTransferResults:
"""
Get transfers that completed on this specific worker, including
requests whose remote KV load failed.
The scheduler process (via the MultiprocExecutor) will use this output
to track which workers are done.
"""
finished_sending_reqs, finished_recving_reqs = self.get_finished()
failed_recving_reqs: set[ReqId] = set()
while True:
try:
failed_recving_reqs.add(self._failed_recv_reqs.get_nowait())
except queue.Empty:
break
return KVConnectorTransferResults(
finished_sending=set(finished_sending_reqs or ()),
finished_recving=set(finished_recving_reqs or ()),
failed_recving=failed_recving_reqs,
)
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""Return transfer stats collected since the last call, or None
if nothing has been recorded in this interval."""
@@ -1959,7 +2000,10 @@ class MooncakeConnectorWorker:
# request is waiting on a load, and reporting one here would trip
# the scheduler's `assert req_id in self.requests`.
continue
self._invalid_block_ids.put(invalid)
if self._is_hma_required:
self._failed_recv_reqs.put(pull_meta.d_req_id)
else:
self._invalid_block_ids.put(invalid)
self.finished_recving_reqs.add(pull_meta.d_req_id)
if failed:
@@ -27,6 +27,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
KVConnectorWorkerMetadata,
SupportsHMA,
)
@@ -359,6 +360,14 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
return self.connector_worker.get_finished(finished_req_ids, metadata)
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
return self.connector_worker.get_transfer_results(finished_req_ids, metadata)
def get_block_ids_with_load_errors(self) -> set[int]:
assert self.connector_worker is not None
return self.connector_worker.get_block_ids_with_load_errors()
@@ -36,6 +36,9 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.distributed.kv_events import BlockStored
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorTransferResults,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
ExternalCachedBlockPool,
@@ -1265,6 +1268,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
record_operation: Callable[..., None] | None = None,
request_queue: queue.Queue[Any] | None = None,
group_participates: Sequence[bool] | None = None,
is_hma_required: bool = False,
):
super().__init__(
store,
@@ -1284,6 +1288,10 @@ class KVCacheStoreRecvingThread(KVTransferThread):
# _invalid_block_ids can be access by both the Worker and RecvingThread
self._invalid_block_ids_lock = threading.Lock()
self._invalid_block_ids: set[int] = set()
# With HMA the scheduler tracks a single merged group while block IDs
# are only unique within a group, so failures are reported per request.
self._is_hma_required = is_hma_required
self._failed_requests: set[str] = set()
self.disk_offload_buffer_budget_bytes = disk_offload_buffer_budget_bytes
self.usable_disk_offload_buffer_budget_bytes = (
None
@@ -1298,6 +1306,22 @@ class KVCacheStoreRecvingThread(KVTransferThread):
with self._invalid_block_ids_lock:
self._invalid_block_ids.update(block_ids)
def set_failed_request(self, req_id: str):
with self.done_task_lock:
self._failed_requests.add(req_id)
def get_and_clear_failed_requests(self) -> set[str]:
with self.done_task_lock:
failed = self._failed_requests.copy()
self._failed_requests.clear()
return failed
def _report_load_error(self, req_id: str, block_ids: list[int]) -> None:
if self._is_hma_required:
self.set_failed_request(req_id)
else:
self._add_load_error_block_ids(block_ids)
def get_and_clear_block_ids_with_load_errors(self) -> set[int]:
with self._invalid_block_ids_lock:
invalid_block_ids = self._invalid_block_ids.copy()
@@ -1392,7 +1416,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
# Mark every block: we skip the whole request, and the
# tp_rank rotation means oversized_key isn't necessarily
# the first block in the request's original order.
self._add_load_error_block_ids(block_id_list_c)
self._report_load_error(req_id, block_id_list_c)
oversized_key_bytes = _estimate_disk_offload_staging_bytes(
size_list_c[oversized_key_index]
)
@@ -1465,8 +1489,8 @@ class KVCacheStoreRecvingThread(KVTransferThread):
num_failed_keys=len(failed),
)
if failed:
self._add_load_error_block_ids(
[block_id for _, _, block_id in failed]
self._report_load_error(
req_id, [block_id for _, _, block_id in failed]
)
logger.warning(
"Failed to get %d Mooncake keys from sub-batch "
@@ -1477,7 +1501,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
)
break
except Exception as e:
self._add_load_error_block_ids(current_batch_block_ids)
self._report_load_error(req_id, current_batch_block_ids)
self._record_operation(
"load_get",
load_get_start,
@@ -1551,6 +1575,13 @@ class MooncakeStoreWorker:
and not self.can_put
)
self.cache_config = vllm_config.cache_config
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.transfer_groups
)
)
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
@@ -2096,6 +2127,7 @@ class MooncakeStoreWorker:
group.kv_cache_spec.prefix_cacheable
for group in self._kv_cache_groups
],
is_hma_required=self._is_hma_required,
)
recv_thread.name = f"KVCacheStoreRecvingThread-{i}"
recv_thread.start()
@@ -2185,6 +2217,26 @@ class MooncakeStoreWorker:
block_ids |= recv_thread.get_and_clear_block_ids_with_load_errors()
return block_ids
def get_transfer_results(
self, finished_req_ids: set[str], meta: MooncakeStoreConnectorMetadata
) -> KVConnectorTransferResults:
"""Get completed sends/recvs plus requests whose remote KV load failed."""
done_sending, done_recving = self.get_finished(finished_req_ids, meta)
if self._capacity_only:
return KVConnectorTransferResults(done_sending, done_recving)
failed_recving: set[str] = set()
if self.load_async:
for recv_thread in self.kv_recv_threads:
failed_recving |= recv_thread.get_and_clear_failed_requests()
return KVConnectorTransferResults(
finished_sending=done_sending,
finished_recving=done_recving,
failed_recving=failed_recving,
)
def _record_kv_connector_operation(
self,
operation: str,
+14 -38
View File
@@ -3027,11 +3027,6 @@ class Scheduler(SchedulerInterface):
For observability, it also accumulates the total number of tokens that
will need to be recomputed across all affected requests.
A hybrid model spreads one token span over KV cache groups that disagree
on block size, so a block index maps to a different token position in
each group and there is no single longest valid prefix. Those requests
recompute in full instead.
Args:
requests: The set of requests to scan for invalid blocks.
invalid_block_ids: IDs of invalid blocks.
@@ -3055,49 +3050,19 @@ class Scheduler(SchedulerInterface):
# these requests must be rescheduled, but only the first will recompute
# it. This set tracks blocks already marked for recomputation.
marked_invalid_block_ids: set[int] = set()
# Positions a group holds no KV for -- skipped sliding-window blocks,
# and every Mamba block but the aligned snapshot -- all point at this
# one block, shared by every request. It carries no request's data, so
# it can neither fail for a request nor be evicted on its behalf.
null_block_id = self.kv_cache_manager.block_pool.null_block.block_id
for request in requests:
is_affected = False
marked_invalid_block = False
req_id = request.request_id
# One block-id list per KV cache group.
req_block_ids_per_group = self.kv_cache_manager.get_block_ids(req_id)
# One block-id list per KV cache group; single-group layouts are
# enforced by _handle_invalid_blocks.
(req_block_ids,) = self.kv_cache_manager.get_block_ids(req_id)
# We iterate only over blocks that may contain externally computed
# tokens
req_num_computed_tokens = (
request.num_computed_tokens - num_scheduled_tokens.get(req_id, 0)
)
if len(req_block_ids_per_group) > 1:
# No single truncation point exists across groups, so the whole
# request is recomputed. Blocks are still marked, so a request
# sharing one of them is rescheduled too.
request_invalid_block_ids = {
block_id
for req_block_ids in req_block_ids_per_group
for block_id in req_block_ids
if block_id != null_block_id and block_id in invalid_block_ids
}
if request_invalid_block_ids:
marked_invalid_block_ids |= request_invalid_block_ids
total_affected_tokens += req_num_computed_tokens
request.num_computed_tokens = 0
if evict_blocks:
for req_block_ids in req_block_ids_per_group:
blocks_to_evict.update(
block_id
for block_id in req_block_ids
if block_id != null_block_id
)
affected_req_ids.add(req_id)
continue
(req_block_ids,) = req_block_ids_per_group
req_num_computed_blocks = (
req_num_computed_tokens + self.block_size - 1
) // self.block_size
@@ -3175,6 +3140,17 @@ class Scheduler(SchedulerInterface):
Returns:
Set of affected request IDs to skip in update_from_output main loop.
"""
if len(self.kv_cache_config.kv_cache_groups) > 1:
# Block IDs are only unique within a group, so a flat set of
# invalid block IDs cannot be mapped back to requests.
raise RuntimeError(
"A KV connector reported block-level load failures "
"(invalid_block_ids) on a layout with multiple KV cache "
"groups, where block IDs are only unique within a group. "
"Connectors must report failed requests via "
"KVConnectorTransferResults.failed_recving instead."
)
should_fail = not self.recompute_kv_load_failures
# handle async KV loads (not cached yet, evict_blocks=False)