[Bugfix][Mooncake] Report request-level KV load failures under HMA (#56855)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -2,21 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Literal
|
||||
from unittest.mock import Mock, patch
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
)
|
||||
from vllm.v1.request import FinishReason, Request, RequestStatus
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
from .utils import (
|
||||
create_model_runner_output,
|
||||
@@ -44,18 +35,6 @@ def scheduler():
|
||||
return create_scheduler(vllm_config)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hybrid_scheduler():
|
||||
"""Scheduler for a hybrid model: full attention plus a Mamba group."""
|
||||
vllm_config = create_vllm_config(kv_load_failure_policy="recompute")
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
mamba_enabled=True,
|
||||
num_blocks=10000,
|
||||
)
|
||||
return create_scheduler(vllm_config, kv_cache_config=kv_cache_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_prompt_blocks,num_external_computed_blocks,invalid_block_idxs",
|
||||
[
|
||||
@@ -361,317 +340,19 @@ def test_async_progressive_load_failure(
|
||||
assert scheduler.connector.get_num_new_matched_tokens.call_count == 1
|
||||
|
||||
|
||||
def _schedule_hybrid_async_load(
|
||||
scheduler: Scheduler, num_prompt_blocks: int, num_external_computed_blocks: int
|
||||
) -> Request:
|
||||
"""Put one request into an async external load on a hybrid model."""
|
||||
request = create_request(num_tokens=num_prompt_blocks * scheduler.block_size)
|
||||
scheduler.add_request(request=request)
|
||||
@pytest.mark.cpu_test
|
||||
def test_block_level_load_failure_fails_closed_with_multiple_groups():
|
||||
"""Block-level invalid-block reporting is rejected on multi-group layouts.
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(
|
||||
{request.request_id: num_external_computed_blocks * scheduler.block_size},
|
||||
async_load=True,
|
||||
)
|
||||
)
|
||||
scheduler.connector.take_events.return_value = ()
|
||||
return request
|
||||
|
||||
|
||||
def _report_invalid_blocks(
|
||||
scheduler: Scheduler, scheduler_output, invalid_block_ids: set[int]
|
||||
) -> None:
|
||||
scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
create_model_runner_output(
|
||||
reqs=[],
|
||||
finished_recving=set(),
|
||||
invalid_block_ids=invalid_block_ids,
|
||||
use_eos=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failed_group_idx", [0, 1])
|
||||
def test_hybrid_load_failure_recomputes_whole_request(
|
||||
hybrid_scheduler: Scheduler, failed_group_idx: int
|
||||
):
|
||||
"""A hybrid model must recover, not crash.
|
||||
|
||||
Its groups sit on different block grids, so there is no single truncation
|
||||
point; the request restarts from scratch instead.
|
||||
Block IDs are only unique within a group, so the scheduler cannot map
|
||||
them back to requests; connectors must report failed_recving instead.
|
||||
"""
|
||||
num_prompt_blocks, num_external_computed_blocks = 10, 9
|
||||
request = _schedule_hybrid_async_load(
|
||||
hybrid_scheduler, num_prompt_blocks, num_external_computed_blocks
|
||||
)
|
||||
scheduler_output = hybrid_scheduler.schedule()
|
||||
|
||||
block_ids_per_group = hybrid_scheduler.kv_cache_manager.get_block_ids(
|
||||
request.request_id
|
||||
)
|
||||
assert len(block_ids_per_group) == 2, "expected a hybrid (multi-group) layout"
|
||||
null_block_id = hybrid_scheduler.kv_cache_manager.block_pool.null_block.block_id
|
||||
# The Mamba group backs only its aligned snapshot with a real block, so
|
||||
# pick a block that actually belongs to this request.
|
||||
failed_block = next(
|
||||
block_id
|
||||
for block_id in block_ids_per_group[failed_group_idx]
|
||||
if block_id != null_block_id
|
||||
)
|
||||
|
||||
_report_invalid_blocks(hybrid_scheduler, scheduler_output, {failed_block})
|
||||
|
||||
assert request.num_computed_tokens == 0
|
||||
assert hybrid_scheduler.failed_recving_kv_req_ids == {request.request_id}
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
|
||||
|
||||
def test_hybrid_load_failure_ignores_null_block(hybrid_scheduler: Scheduler):
|
||||
"""The null block is shared by every request and holds no KV, so reporting
|
||||
it must not restart anyone."""
|
||||
num_prompt_blocks, num_external_computed_blocks = 10, 9
|
||||
request = _schedule_hybrid_async_load(
|
||||
hybrid_scheduler, num_prompt_blocks, num_external_computed_blocks
|
||||
)
|
||||
scheduler_output = hybrid_scheduler.schedule()
|
||||
|
||||
null_block_id = hybrid_scheduler.kv_cache_manager.block_pool.null_block.block_id
|
||||
_, mamba_block_ids = hybrid_scheduler.kv_cache_manager.get_block_ids(
|
||||
request.request_id
|
||||
)
|
||||
# The Mamba group parks its unaligned positions on the null block, so a
|
||||
# request that never failed would otherwise be restarted by this report.
|
||||
assert mamba_block_ids[0] == null_block_id
|
||||
|
||||
num_computed_tokens = request.num_computed_tokens
|
||||
_report_invalid_blocks(hybrid_scheduler, scheduler_output, {null_block_id})
|
||||
|
||||
assert request.num_computed_tokens == num_computed_tokens
|
||||
assert not hybrid_scheduler.failed_recving_kv_req_ids
|
||||
|
||||
|
||||
def _make_deepseek_v4_five_group_config(num_blocks: int) -> KVCacheConfig:
|
||||
"""Build the scheduler-side projection of V4-Flash's packed KV layout."""
|
||||
|
||||
def mla_spec() -> MLAAttentionSpec:
|
||||
return MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=4,
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
|
||||
def swa_spec(
|
||||
*, block_size: int, sliding_window: int, head_size: int
|
||||
) -> SlidingWindowMLASpec:
|
||||
return SlidingWindowMLASpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=head_size,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sliding_window,
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
|
||||
# DeepSeek-V4 packs these into five physical groups. The two SWA groups
|
||||
# have identical scheduling semantics but are split to align layer tuples.
|
||||
groups = [
|
||||
KVCacheGroupSpec(["full_mla_and_indexer"], mla_spec()),
|
||||
KVCacheGroupSpec(
|
||||
["c4_state"],
|
||||
swa_spec(block_size=4, sliding_window=8, head_size=2048),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["c128_state"],
|
||||
swa_spec(block_size=8, sliding_window=128, head_size=2048),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["swa.0"],
|
||||
swa_spec(block_size=64, sliding_window=512, head_size=512),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["swa.1"],
|
||||
swa_spec(block_size=64, sliding_window=512, head_size=512),
|
||||
),
|
||||
]
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
|
||||
|
||||
def _create_deepseek_v4_scheduler(
|
||||
kv_load_failure_policy: Literal["recompute", "fail"] = "recompute",
|
||||
) -> Scheduler:
|
||||
num_blocks = 10000
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=256,
|
||||
max_num_batched_tokens=4096,
|
||||
max_model_len=8192,
|
||||
# max_model_len only reaches SchedulerConfig; opt-125m still caps the
|
||||
# model at 2048, which is below the 9-block requests used here.
|
||||
hf_overrides={"max_position_embeddings": 8192},
|
||||
kv_load_failure_policy=kv_load_failure_policy,
|
||||
)
|
||||
return create_scheduler(
|
||||
vllm_config = create_vllm_config()
|
||||
scheduler = create_scheduler(
|
||||
vllm_config,
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_config=_make_deepseek_v4_five_group_config(num_blocks),
|
||||
hash_block_size=4,
|
||||
kv_cache_config=make_kv_cache_config(block_size=16, swa_enabled=True),
|
||||
)
|
||||
assert len(scheduler.kv_cache_config.kv_cache_groups) == 2
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def deepseek_v4_scheduler() -> Scheduler:
|
||||
return _create_deepseek_v4_scheduler()
|
||||
|
||||
|
||||
def _schedule_deepseek_v4_async_load(
|
||||
scheduler: Scheduler,
|
||||
) -> tuple[Request, SchedulerOutput]:
|
||||
num_external_tokens = 8 * scheduler.block_size
|
||||
request = create_request(num_tokens=9 * scheduler.block_size, block_size=4)
|
||||
scheduler.add_request(request)
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(
|
||||
{request.request_id: num_external_tokens}, async_load=True
|
||||
)
|
||||
)
|
||||
scheduler.connector.take_events.return_value = ()
|
||||
return request, scheduler.schedule()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failed_group_idx", range(5))
|
||||
def test_deepseek_v4_load_failure_recomputes_whole_request(
|
||||
deepseek_v4_scheduler: Scheduler, failed_group_idx: int
|
||||
):
|
||||
"""A failed load in any V4-Flash KV group must not kill EngineCore."""
|
||||
request, scheduler_output = _schedule_deepseek_v4_async_load(deepseek_v4_scheduler)
|
||||
|
||||
block_ids_per_group = deepseek_v4_scheduler.kv_cache_manager.get_block_ids(
|
||||
request.request_id
|
||||
)
|
||||
assert len(block_ids_per_group) == 5
|
||||
null_block_id = (
|
||||
deepseek_v4_scheduler.kv_cache_manager.block_pool.null_block.block_id
|
||||
)
|
||||
failed_block_id = next(
|
||||
block_id
|
||||
for block_id in block_ids_per_group[failed_group_idx]
|
||||
if block_id != null_block_id
|
||||
)
|
||||
|
||||
deepseek_v4_scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
create_model_runner_output(
|
||||
reqs=[],
|
||||
finished_recving=set(),
|
||||
invalid_block_ids={failed_block_id},
|
||||
use_eos=True,
|
||||
),
|
||||
)
|
||||
|
||||
assert request.num_computed_tokens == 0
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert deepseek_v4_scheduler.failed_recving_kv_req_ids == {request.request_id}
|
||||
|
||||
|
||||
def test_deepseek_v4_load_failure_ignores_shared_null_block(
|
||||
deepseek_v4_scheduler: Scheduler,
|
||||
):
|
||||
"""The shared null block contains no request KV and cannot fail a load."""
|
||||
request, scheduler_output = _schedule_deepseek_v4_async_load(deepseek_v4_scheduler)
|
||||
null_block_id = (
|
||||
deepseek_v4_scheduler.kv_cache_manager.block_pool.null_block.block_id
|
||||
)
|
||||
block_ids_per_group = deepseek_v4_scheduler.kv_cache_manager.get_block_ids(
|
||||
request.request_id
|
||||
)
|
||||
assert any(null_block_id in group for group in block_ids_per_group[1:])
|
||||
num_computed_tokens = request.num_computed_tokens
|
||||
|
||||
deepseek_v4_scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
create_model_runner_output(
|
||||
reqs=[],
|
||||
finished_recving=set(),
|
||||
invalid_block_ids={null_block_id},
|
||||
use_eos=True,
|
||||
),
|
||||
)
|
||||
|
||||
assert request.num_computed_tokens == num_computed_tokens
|
||||
assert not deepseek_v4_scheduler.failed_recving_kv_req_ids
|
||||
|
||||
|
||||
def test_deepseek_v4_sync_load_failure_fails_and_evicts_real_blocks():
|
||||
"""Fail policy must terminate cleanly and never evict the null block."""
|
||||
scheduler = _create_deepseek_v4_scheduler(kv_load_failure_policy="fail")
|
||||
num_external_tokens = 8 * scheduler.block_size
|
||||
request = create_request(num_tokens=9 * scheduler.block_size, block_size=4)
|
||||
scheduler.add_request(request)
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(
|
||||
{request.request_id: num_external_tokens}, async_load=False
|
||||
)
|
||||
)
|
||||
scheduler.connector.request_finished.return_value = (False, None)
|
||||
scheduler.connector.take_events.return_value = ()
|
||||
scheduler_output = scheduler.schedule()
|
||||
|
||||
block_ids_per_group = scheduler.kv_cache_manager.get_block_ids(request.request_id)
|
||||
assert len(block_ids_per_group) == 5
|
||||
null_block_id = scheduler.kv_cache_manager.block_pool.null_block.block_id
|
||||
real_block_ids = {
|
||||
block_id
|
||||
for group_block_ids in block_ids_per_group
|
||||
for block_id in group_block_ids
|
||||
if block_id != null_block_id
|
||||
}
|
||||
failed_block_id = next(
|
||||
block_id for block_id in block_ids_per_group[2] if block_id != null_block_id
|
||||
)
|
||||
|
||||
evicted_block_ids: list[set[int]] = []
|
||||
original_evict_blocks = scheduler.kv_cache_manager.evict_blocks
|
||||
|
||||
def evict_blocks_spy(block_ids: set[int]) -> None:
|
||||
evicted_block_ids.append(set(block_ids))
|
||||
original_evict_blocks(block_ids)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
scheduler.kv_cache_manager,
|
||||
"evict_blocks",
|
||||
side_effect=evict_blocks_spy,
|
||||
),
|
||||
patch.object(scheduler, "_connector_finished", return_value=(False, None)),
|
||||
):
|
||||
outputs = scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
create_model_runner_output(
|
||||
[request],
|
||||
invalid_block_ids={failed_block_id},
|
||||
use_eos=True,
|
||||
),
|
||||
)
|
||||
|
||||
assert request.status == RequestStatus.FINISHED_ERROR
|
||||
assert request.get_finished_reason() == FinishReason.ERROR
|
||||
assert request.request_id not in scheduler.requests
|
||||
assert evicted_block_ids == [real_block_ids]
|
||||
assert null_block_id not in evicted_block_ids[0]
|
||||
|
||||
engine_outputs = next(iter(outputs.values()))
|
||||
assert len(engine_outputs.outputs) == 1
|
||||
assert engine_outputs.outputs[0].request_id == request.request_id
|
||||
assert engine_outputs.outputs[0].finish_reason == FinishReason.ERROR
|
||||
with pytest.raises(RuntimeError, match="failed_recving"):
|
||||
scheduler._handle_invalid_blocks({1}, {})
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im
|
||||
MooncakeConnectorMetadata,
|
||||
MooncakeConnectorScheduler,
|
||||
MooncakeXferMetadata,
|
||||
PullReqMeta,
|
||||
SendBlockMeta,
|
||||
TransferRegion,
|
||||
)
|
||||
@@ -436,3 +437,79 @@ def test_request_finished_with_hma_groups():
|
||||
assert stored_blocks[0] == fa_blocks
|
||||
# SW: clipped to last 9 blocks (sw_size=128, block_size=16 → 8+1=9)
|
||||
assert stored_blocks[1] == sw_blocks[-9:]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Worker-side load-failure reporting (HMA vs non-HMA)
|
||||
# ---------------------------------------------------------------------------
|
||||
def _make_kv_consumer_worker(swa_enabled: bool, disable_hma: bool = False):
|
||||
block_size = 16
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_consumer",
|
||||
block_size=block_size,
|
||||
)
|
||||
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size=block_size, swa_enabled=swa_enabled
|
||||
)
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
connector = MooncakeConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
return connector.connector_worker
|
||||
|
||||
|
||||
def _make_pull_meta(d_req_id: str, local_block_ids: list[list[int]]) -> PullReqMeta:
|
||||
return PullReqMeta(
|
||||
d_req_id=d_req_id,
|
||||
transfer_id="tx-1",
|
||||
local_block_ids=local_block_ids,
|
||||
remote_engine_id="engine-0",
|
||||
remote_bootstrap_addr="127.0.0.1:1234",
|
||||
)
|
||||
|
||||
|
||||
# The tests below construct a MooncakeConnectorWorker, whose __init__
|
||||
# requires an accelerator device, so (like the worker tests in
|
||||
# test_mooncake_connector.py) they run on the GPU KV-connectors job only.
|
||||
@pytest.mark.parametrize(
|
||||
"swa_enabled,disable_hma,expected_is_hma",
|
||||
[
|
||||
(True, False, True), # SWA group present, HMA enabled
|
||||
(True, True, False), # SWA group present, but HMA disabled
|
||||
(False, False, False), # FA only, HMA not needed
|
||||
],
|
||||
)
|
||||
def test_worker_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
|
||||
"""Worker-side _is_hma_required mirrors the scheduler's derivation."""
|
||||
worker = _make_kv_consumer_worker(swa_enabled, disable_hma)
|
||||
assert worker._is_hma_required is expected_is_hma
|
||||
|
||||
|
||||
def test_worker_failed_recv_reports_request_level_failure_with_hma():
|
||||
"""With HMA, load failures report the request, not ambiguous block IDs."""
|
||||
worker = _make_kv_consumer_worker(swa_enabled=True)
|
||||
assert worker._is_hma_required
|
||||
|
||||
pull_meta = _make_pull_meta("d-req-1", [[1, 2], [3, 4]])
|
||||
worker._handle_failed_recv({"p-req-1": pull_meta}, ["p-req-1"], "boom")
|
||||
|
||||
assert worker.get_block_ids_with_load_errors() == set()
|
||||
results = worker.get_transfer_results()
|
||||
assert results.failed_recving == {"d-req-1"}
|
||||
assert results.finished_recving == {"d-req-1"}
|
||||
|
||||
|
||||
def test_worker_failed_recv_reports_block_ids_without_hma():
|
||||
"""Without HMA, load failures keep reporting block-level errors."""
|
||||
worker = _make_kv_consumer_worker(swa_enabled=False)
|
||||
assert not worker._is_hma_required
|
||||
|
||||
pull_meta = _make_pull_meta("d-req-1", [[1, 2], [3, 4]])
|
||||
worker._handle_failed_recv({"p-req-1": pull_meta}, ["p-req-1"], "boom")
|
||||
|
||||
assert worker.get_block_ids_with_load_errors() == {1, 2, 3, 4}
|
||||
results = worker.get_transfer_results()
|
||||
assert results.failed_recving == set()
|
||||
assert results.finished_recving == {"d-req-1"}
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_events import BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
connector as mooncake_store_connector,
|
||||
@@ -245,6 +246,38 @@ def test_worker_methods_delegate_to_store_worker():
|
||||
assert invalid_block_ids == {3, 4}
|
||||
|
||||
|
||||
def test_get_transfer_results_delegates_to_store_worker():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
metadata = MooncakeStoreConnectorMetadata(set(), set())
|
||||
finished_req_ids = {"req-1"}
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker_cls,
|
||||
):
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
worker = mock_worker_cls.return_value
|
||||
transfer_results = KVConnectorTransferResults(
|
||||
finished_sending={"req-1"},
|
||||
finished_recving={"req-2"},
|
||||
failed_recving={"req-3"},
|
||||
)
|
||||
worker.get_transfer_results.return_value = transfer_results
|
||||
connector.bind_connector_metadata(metadata)
|
||||
|
||||
result = connector.get_transfer_results(finished_req_ids)
|
||||
|
||||
worker.get_transfer_results.assert_called_once_with(finished_req_ids, metadata)
|
||||
assert result is transfer_results
|
||||
|
||||
|
||||
def test_get_kv_connector_kv_cache_events_returns_none_when_disabled():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
@@ -177,6 +177,7 @@ def _make_store_recving_thread(
|
||||
*,
|
||||
tp_rank: int = 0,
|
||||
disk_offload_buffer_budget_bytes: int | None = None,
|
||||
is_hma_required: bool = False,
|
||||
) -> mooncake_store_worker.KVCacheStoreRecvingThread:
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
@@ -199,6 +200,7 @@ def _make_store_recving_thread(
|
||||
ready_event=threading.Event(),
|
||||
coord=coord,
|
||||
disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes,
|
||||
is_hma_required=is_hma_required,
|
||||
)
|
||||
thread.request_queue.task_done = MagicMock()
|
||||
return thread
|
||||
@@ -303,6 +305,7 @@ def _make_vllm_config(
|
||||
kv_role: str = "kv_both",
|
||||
pipeline_parallel_size: int = 1,
|
||||
kv_cache_layout: KVCacheLayout = KVCacheLayout.LBHNC,
|
||||
disable_hybrid_kv_cache_manager: bool = True,
|
||||
) -> SimpleNamespace:
|
||||
cache_config = SimpleNamespace(block_size=16, num_gpu_blocks=10)
|
||||
cache_config.get_resolved_kv_cache_layout = lambda: kv_cache_layout
|
||||
@@ -319,6 +322,9 @@ def _make_vllm_config(
|
||||
kv_role=kv_role, extra_config=extra_config
|
||||
),
|
||||
cache_config=cache_config,
|
||||
scheduler_config=SimpleNamespace(
|
||||
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager
|
||||
),
|
||||
kv_events_config=SimpleNamespace(enable_kv_cache_events=False),
|
||||
speculative_config=None,
|
||||
)
|
||||
@@ -2147,6 +2153,48 @@ def test_recv_thread_reports_unsplittable_key_larger_than_budget():
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2}
|
||||
|
||||
|
||||
def test_recv_thread_partial_load_failure_reports_block_ids_without_hma():
|
||||
store = MagicMock()
|
||||
# First key fails, second succeeds.
|
||||
store.batch_get_into_multi_buffers.return_value = [-1, 0]
|
||||
thread = _make_store_recving_thread(store)
|
||||
|
||||
thread._handle_request(_make_load_req("req-a", [b"h0", b"h1"], token_len=32))
|
||||
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {0}
|
||||
assert thread.get_and_clear_failed_requests() == set()
|
||||
assert thread.get_and_clear_finished_requests() == {"req-a"}
|
||||
|
||||
|
||||
def test_recv_thread_partial_load_failure_reports_request_with_hma():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [-1, 0]
|
||||
thread = _make_store_recving_thread(store, is_hma_required=True)
|
||||
|
||||
thread._handle_request(_make_load_req("req-a", [b"h0", b"h1"], token_len=32))
|
||||
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == set()
|
||||
assert thread.get_and_clear_failed_requests() == {"req-a"}
|
||||
assert thread.get_and_clear_finished_requests() == {"req-a"}
|
||||
|
||||
|
||||
def test_recv_thread_oversized_key_reports_request_with_hma():
|
||||
store = MagicMock()
|
||||
thread = _make_store_recving_thread(
|
||||
store,
|
||||
tp_rank=2,
|
||||
disk_offload_buffer_budget_bytes=_DISK_OFFLOAD_BUDGET_TOO_SMALL,
|
||||
is_hma_required=True,
|
||||
)
|
||||
|
||||
thread._handle_request(_make_load_req("req-a", [b"a0", b"a1", b"a2"], token_len=48))
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 0
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == set()
|
||||
assert thread.get_and_clear_failed_requests() == {"req-a"}
|
||||
assert thread.get_and_clear_finished_requests() == {"req-a"}
|
||||
|
||||
|
||||
def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
@@ -2172,6 +2220,39 @@ def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch):
|
||||
assert [db.block_size for db in store_worker.token_dbs] == [800, 800]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("disable_hybrid_kv_cache_manager", "expected_is_hma_required"),
|
||||
[
|
||||
(True, False),
|
||||
(False, True),
|
||||
],
|
||||
)
|
||||
def test_worker_is_hma_required_from_kv_cache_groups(
|
||||
monkeypatch, disable_hybrid_kv_cache_manager, expected_is_hma_required
|
||||
):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
worker.MooncakeStoreConfig,
|
||||
"load_from_config",
|
||||
staticmethod(lambda: _make_config()),
|
||||
)
|
||||
vllm_config = _make_vllm_config(
|
||||
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager
|
||||
)
|
||||
vllm_config.cache_config.block_size = 800
|
||||
vllm_config.cache_config.enable_prefix_caching = True
|
||||
vllm_config.cache_config.prefix_match_unit = None
|
||||
|
||||
store_worker = worker.MooncakeStoreWorker(
|
||||
vllm_config, _make_qsa_hybrid_kv_cache_config()
|
||||
)
|
||||
|
||||
assert store_worker._is_hma_required is expected_is_hma_required
|
||||
|
||||
|
||||
def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
@@ -3345,6 +3426,7 @@ def _make_bare_worker(
|
||||
worker.can_put = kv_role in ("kv_producer", "kv_both") or save_decode_cache
|
||||
worker._capacity_only = False
|
||||
worker.block_size = block_size
|
||||
worker._is_hma_required = False
|
||||
worker.tp_rank = 0
|
||||
worker.enable_kv_events = False
|
||||
worker.load_async = True
|
||||
|
||||
@@ -155,7 +155,6 @@ def create_scheduler(
|
||||
vllm_config: VllmConfig,
|
||||
num_blocks: int = 10000,
|
||||
kv_cache_config: KVCacheConfig | None = None,
|
||||
hash_block_size: int | None = None,
|
||||
) -> Scheduler | AsyncScheduler:
|
||||
"""Initialize Scheduler For Testing."""
|
||||
block_size = vllm_config.cache_config.block_size
|
||||
@@ -186,7 +185,6 @@ def create_scheduler(
|
||||
log_stats=True,
|
||||
structured_output_manager=StructuredOutputManager(vllm_config),
|
||||
block_size=block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
@@ -587,6 +588,13 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_finished()
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> KVConnectorTransferResults:
|
||||
"""Get finished and failed sends/recvs from the worker."""
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_transfer_results()
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Get block IDs whose remote KV load failed."""
|
||||
assert self.connector_worker is not None
|
||||
@@ -1046,6 +1054,16 @@ class MooncakeConnectorWorker:
|
||||
self.finished_recving_reqs: set[ReqId] = set()
|
||||
# Written from the receiver loop, drained from the worker thread.
|
||||
self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue()
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
# Block IDs are only unique within a group; with HMA the scheduler
|
||||
# tracks a single merged group, so failures are reported per request.
|
||||
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
|
||||
|
||||
self.xfer_stats = MooncakeKVConnectorStats()
|
||||
|
||||
@@ -1866,6 +1884,29 @@ class MooncakeConnectorWorker:
|
||||
|
||||
return finished_sending_reqs or None, finished_recving_reqs or None
|
||||
|
||||
def get_transfer_results(self) -> KVConnectorTransferResults:
|
||||
"""
|
||||
Get transfers that completed on this specific worker, including
|
||||
requests whose remote KV load failed.
|
||||
|
||||
The scheduler process (via the MultiprocExecutor) will use this output
|
||||
to track which workers are done.
|
||||
"""
|
||||
finished_sending_reqs, finished_recving_reqs = self.get_finished()
|
||||
|
||||
failed_recving_reqs: set[ReqId] = set()
|
||||
while True:
|
||||
try:
|
||||
failed_recving_reqs.add(self._failed_recv_reqs.get_nowait())
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
return KVConnectorTransferResults(
|
||||
finished_sending=set(finished_sending_reqs or ()),
|
||||
finished_recving=set(finished_recving_reqs or ()),
|
||||
failed_recving=failed_recving_reqs,
|
||||
)
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""Return transfer stats collected since the last call, or None
|
||||
if nothing has been recorded in this interval."""
|
||||
@@ -1959,7 +2000,10 @@ class MooncakeConnectorWorker:
|
||||
# request is waiting on a load, and reporting one here would trip
|
||||
# the scheduler's `assert req_id in self.requests`.
|
||||
continue
|
||||
self._invalid_block_ids.put(invalid)
|
||||
if self._is_hma_required:
|
||||
self._failed_recv_reqs.put(pull_meta.d_req_id)
|
||||
else:
|
||||
self._invalid_block_ids.put(invalid)
|
||||
self.finished_recving_reqs.add(pull_meta.d_req_id)
|
||||
|
||||
if failed:
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
KVConnectorWorkerMetadata,
|
||||
SupportsHMA,
|
||||
)
|
||||
@@ -359,6 +360,14 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
return self.connector_worker.get_finished(finished_req_ids, metadata)
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> KVConnectorTransferResults:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
return self.connector_worker.get_transfer_results(finished_req_ids, metadata)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_block_ids_with_load_errors()
|
||||
|
||||
@@ -36,6 +36,9 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.distributed.kv_events import BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorTransferResults,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
|
||||
ExternalCachedBlockPool,
|
||||
@@ -1265,6 +1268,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
record_operation: Callable[..., None] | None = None,
|
||||
request_queue: queue.Queue[Any] | None = None,
|
||||
group_participates: Sequence[bool] | None = None,
|
||||
is_hma_required: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
store,
|
||||
@@ -1284,6 +1288,10 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
# _invalid_block_ids can be access by both the Worker and RecvingThread
|
||||
self._invalid_block_ids_lock = threading.Lock()
|
||||
self._invalid_block_ids: set[int] = set()
|
||||
# With HMA the scheduler tracks a single merged group while block IDs
|
||||
# are only unique within a group, so failures are reported per request.
|
||||
self._is_hma_required = is_hma_required
|
||||
self._failed_requests: set[str] = set()
|
||||
self.disk_offload_buffer_budget_bytes = disk_offload_buffer_budget_bytes
|
||||
self.usable_disk_offload_buffer_budget_bytes = (
|
||||
None
|
||||
@@ -1298,6 +1306,22 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
with self._invalid_block_ids_lock:
|
||||
self._invalid_block_ids.update(block_ids)
|
||||
|
||||
def set_failed_request(self, req_id: str):
|
||||
with self.done_task_lock:
|
||||
self._failed_requests.add(req_id)
|
||||
|
||||
def get_and_clear_failed_requests(self) -> set[str]:
|
||||
with self.done_task_lock:
|
||||
failed = self._failed_requests.copy()
|
||||
self._failed_requests.clear()
|
||||
return failed
|
||||
|
||||
def _report_load_error(self, req_id: str, block_ids: list[int]) -> None:
|
||||
if self._is_hma_required:
|
||||
self.set_failed_request(req_id)
|
||||
else:
|
||||
self._add_load_error_block_ids(block_ids)
|
||||
|
||||
def get_and_clear_block_ids_with_load_errors(self) -> set[int]:
|
||||
with self._invalid_block_ids_lock:
|
||||
invalid_block_ids = self._invalid_block_ids.copy()
|
||||
@@ -1392,7 +1416,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
# Mark every block: we skip the whole request, and the
|
||||
# tp_rank rotation means oversized_key isn't necessarily
|
||||
# the first block in the request's original order.
|
||||
self._add_load_error_block_ids(block_id_list_c)
|
||||
self._report_load_error(req_id, block_id_list_c)
|
||||
oversized_key_bytes = _estimate_disk_offload_staging_bytes(
|
||||
size_list_c[oversized_key_index]
|
||||
)
|
||||
@@ -1465,8 +1489,8 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
num_failed_keys=len(failed),
|
||||
)
|
||||
if failed:
|
||||
self._add_load_error_block_ids(
|
||||
[block_id for _, _, block_id in failed]
|
||||
self._report_load_error(
|
||||
req_id, [block_id for _, _, block_id in failed]
|
||||
)
|
||||
logger.warning(
|
||||
"Failed to get %d Mooncake keys from sub-batch "
|
||||
@@ -1477,7 +1501,7 @@ class KVCacheStoreRecvingThread(KVTransferThread):
|
||||
)
|
||||
break
|
||||
except Exception as e:
|
||||
self._add_load_error_block_ids(current_batch_block_ids)
|
||||
self._report_load_error(req_id, current_batch_block_ids)
|
||||
self._record_operation(
|
||||
"load_get",
|
||||
load_get_start,
|
||||
@@ -1551,6 +1575,13 @@ class MooncakeStoreWorker:
|
||||
and not self.can_put
|
||||
)
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
@@ -2096,6 +2127,7 @@ class MooncakeStoreWorker:
|
||||
group.kv_cache_spec.prefix_cacheable
|
||||
for group in self._kv_cache_groups
|
||||
],
|
||||
is_hma_required=self._is_hma_required,
|
||||
)
|
||||
recv_thread.name = f"KVCacheStoreRecvingThread-{i}"
|
||||
recv_thread.start()
|
||||
@@ -2185,6 +2217,26 @@ class MooncakeStoreWorker:
|
||||
block_ids |= recv_thread.get_and_clear_block_ids_with_load_errors()
|
||||
return block_ids
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str], meta: MooncakeStoreConnectorMetadata
|
||||
) -> KVConnectorTransferResults:
|
||||
"""Get completed sends/recvs plus requests whose remote KV load failed."""
|
||||
done_sending, done_recving = self.get_finished(finished_req_ids, meta)
|
||||
|
||||
if self._capacity_only:
|
||||
return KVConnectorTransferResults(done_sending, done_recving)
|
||||
|
||||
failed_recving: set[str] = set()
|
||||
if self.load_async:
|
||||
for recv_thread in self.kv_recv_threads:
|
||||
failed_recving |= recv_thread.get_and_clear_failed_requests()
|
||||
|
||||
return KVConnectorTransferResults(
|
||||
finished_sending=done_sending,
|
||||
finished_recving=done_recving,
|
||||
failed_recving=failed_recving,
|
||||
)
|
||||
|
||||
def _record_kv_connector_operation(
|
||||
self,
|
||||
operation: str,
|
||||
|
||||
@@ -3027,11 +3027,6 @@ class Scheduler(SchedulerInterface):
|
||||
For observability, it also accumulates the total number of tokens that
|
||||
will need to be recomputed across all affected requests.
|
||||
|
||||
A hybrid model spreads one token span over KV cache groups that disagree
|
||||
on block size, so a block index maps to a different token position in
|
||||
each group and there is no single longest valid prefix. Those requests
|
||||
recompute in full instead.
|
||||
|
||||
Args:
|
||||
requests: The set of requests to scan for invalid blocks.
|
||||
invalid_block_ids: IDs of invalid blocks.
|
||||
@@ -3055,49 +3050,19 @@ class Scheduler(SchedulerInterface):
|
||||
# these requests must be rescheduled, but only the first will recompute
|
||||
# it. This set tracks blocks already marked for recomputation.
|
||||
marked_invalid_block_ids: set[int] = set()
|
||||
# Positions a group holds no KV for -- skipped sliding-window blocks,
|
||||
# and every Mamba block but the aligned snapshot -- all point at this
|
||||
# one block, shared by every request. It carries no request's data, so
|
||||
# it can neither fail for a request nor be evicted on its behalf.
|
||||
null_block_id = self.kv_cache_manager.block_pool.null_block.block_id
|
||||
for request in requests:
|
||||
is_affected = False
|
||||
marked_invalid_block = False
|
||||
req_id = request.request_id
|
||||
# One block-id list per KV cache group.
|
||||
req_block_ids_per_group = self.kv_cache_manager.get_block_ids(req_id)
|
||||
# One block-id list per KV cache group; single-group layouts are
|
||||
# enforced by _handle_invalid_blocks.
|
||||
(req_block_ids,) = self.kv_cache_manager.get_block_ids(req_id)
|
||||
# We iterate only over blocks that may contain externally computed
|
||||
# tokens
|
||||
req_num_computed_tokens = (
|
||||
request.num_computed_tokens - num_scheduled_tokens.get(req_id, 0)
|
||||
)
|
||||
|
||||
if len(req_block_ids_per_group) > 1:
|
||||
# No single truncation point exists across groups, so the whole
|
||||
# request is recomputed. Blocks are still marked, so a request
|
||||
# sharing one of them is rescheduled too.
|
||||
request_invalid_block_ids = {
|
||||
block_id
|
||||
for req_block_ids in req_block_ids_per_group
|
||||
for block_id in req_block_ids
|
||||
if block_id != null_block_id and block_id in invalid_block_ids
|
||||
}
|
||||
if request_invalid_block_ids:
|
||||
marked_invalid_block_ids |= request_invalid_block_ids
|
||||
total_affected_tokens += req_num_computed_tokens
|
||||
request.num_computed_tokens = 0
|
||||
if evict_blocks:
|
||||
for req_block_ids in req_block_ids_per_group:
|
||||
blocks_to_evict.update(
|
||||
block_id
|
||||
for block_id in req_block_ids
|
||||
if block_id != null_block_id
|
||||
)
|
||||
affected_req_ids.add(req_id)
|
||||
continue
|
||||
|
||||
(req_block_ids,) = req_block_ids_per_group
|
||||
|
||||
req_num_computed_blocks = (
|
||||
req_num_computed_tokens + self.block_size - 1
|
||||
) // self.block_size
|
||||
@@ -3175,6 +3140,17 @@ class Scheduler(SchedulerInterface):
|
||||
Returns:
|
||||
Set of affected request IDs to skip in update_from_output main loop.
|
||||
"""
|
||||
if len(self.kv_cache_config.kv_cache_groups) > 1:
|
||||
# Block IDs are only unique within a group, so a flat set of
|
||||
# invalid block IDs cannot be mapped back to requests.
|
||||
raise RuntimeError(
|
||||
"A KV connector reported block-level load failures "
|
||||
"(invalid_block_ids) on a layout with multiple KV cache "
|
||||
"groups, where block IDs are only unique within a group. "
|
||||
"Connectors must report failed requests via "
|
||||
"KVConnectorTransferResults.failed_recving instead."
|
||||
)
|
||||
|
||||
should_fail = not self.recompute_kv_load_failures
|
||||
|
||||
# handle async KV loads (not cached yet, evict_blocks=False)
|
||||
|
||||
Reference in New Issue
Block a user