[Bugfix][Mooncake] Report request-level KV load failures under HMA (#56855)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -2,21 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Literal
|
||||
from unittest.mock import Mock, patch
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
)
|
||||
from vllm.v1.request import FinishReason, Request, RequestStatus
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
from .utils import (
|
||||
create_model_runner_output,
|
||||
@@ -44,18 +35,6 @@ def scheduler():
|
||||
return create_scheduler(vllm_config)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hybrid_scheduler():
|
||||
"""Scheduler for a hybrid model: full attention plus a Mamba group."""
|
||||
vllm_config = create_vllm_config(kv_load_failure_policy="recompute")
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
mamba_enabled=True,
|
||||
num_blocks=10000,
|
||||
)
|
||||
return create_scheduler(vllm_config, kv_cache_config=kv_cache_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_prompt_blocks,num_external_computed_blocks,invalid_block_idxs",
|
||||
[
|
||||
@@ -361,317 +340,19 @@ def test_async_progressive_load_failure(
|
||||
assert scheduler.connector.get_num_new_matched_tokens.call_count == 1
|
||||
|
||||
|
||||
def _schedule_hybrid_async_load(
|
||||
scheduler: Scheduler, num_prompt_blocks: int, num_external_computed_blocks: int
|
||||
) -> Request:
|
||||
"""Put one request into an async external load on a hybrid model."""
|
||||
request = create_request(num_tokens=num_prompt_blocks * scheduler.block_size)
|
||||
scheduler.add_request(request=request)
|
||||
@pytest.mark.cpu_test
|
||||
def test_block_level_load_failure_fails_closed_with_multiple_groups():
|
||||
"""Block-level invalid-block reporting is rejected on multi-group layouts.
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(
|
||||
{request.request_id: num_external_computed_blocks * scheduler.block_size},
|
||||
async_load=True,
|
||||
)
|
||||
)
|
||||
scheduler.connector.take_events.return_value = ()
|
||||
return request
|
||||
|
||||
|
||||
def _report_invalid_blocks(
|
||||
scheduler: Scheduler, scheduler_output, invalid_block_ids: set[int]
|
||||
) -> None:
|
||||
scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
create_model_runner_output(
|
||||
reqs=[],
|
||||
finished_recving=set(),
|
||||
invalid_block_ids=invalid_block_ids,
|
||||
use_eos=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failed_group_idx", [0, 1])
|
||||
def test_hybrid_load_failure_recomputes_whole_request(
|
||||
hybrid_scheduler: Scheduler, failed_group_idx: int
|
||||
):
|
||||
"""A hybrid model must recover, not crash.
|
||||
|
||||
Its groups sit on different block grids, so there is no single truncation
|
||||
point; the request restarts from scratch instead.
|
||||
Block IDs are only unique within a group, so the scheduler cannot map
|
||||
them back to requests; connectors must report failed_recving instead.
|
||||
"""
|
||||
num_prompt_blocks, num_external_computed_blocks = 10, 9
|
||||
request = _schedule_hybrid_async_load(
|
||||
hybrid_scheduler, num_prompt_blocks, num_external_computed_blocks
|
||||
)
|
||||
scheduler_output = hybrid_scheduler.schedule()
|
||||
|
||||
block_ids_per_group = hybrid_scheduler.kv_cache_manager.get_block_ids(
|
||||
request.request_id
|
||||
)
|
||||
assert len(block_ids_per_group) == 2, "expected a hybrid (multi-group) layout"
|
||||
null_block_id = hybrid_scheduler.kv_cache_manager.block_pool.null_block.block_id
|
||||
# The Mamba group backs only its aligned snapshot with a real block, so
|
||||
# pick a block that actually belongs to this request.
|
||||
failed_block = next(
|
||||
block_id
|
||||
for block_id in block_ids_per_group[failed_group_idx]
|
||||
if block_id != null_block_id
|
||||
)
|
||||
|
||||
_report_invalid_blocks(hybrid_scheduler, scheduler_output, {failed_block})
|
||||
|
||||
assert request.num_computed_tokens == 0
|
||||
assert hybrid_scheduler.failed_recving_kv_req_ids == {request.request_id}
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
|
||||
|
||||
def test_hybrid_load_failure_ignores_null_block(hybrid_scheduler: Scheduler):
|
||||
"""The null block is shared by every request and holds no KV, so reporting
|
||||
it must not restart anyone."""
|
||||
num_prompt_blocks, num_external_computed_blocks = 10, 9
|
||||
request = _schedule_hybrid_async_load(
|
||||
hybrid_scheduler, num_prompt_blocks, num_external_computed_blocks
|
||||
)
|
||||
scheduler_output = hybrid_scheduler.schedule()
|
||||
|
||||
null_block_id = hybrid_scheduler.kv_cache_manager.block_pool.null_block.block_id
|
||||
_, mamba_block_ids = hybrid_scheduler.kv_cache_manager.get_block_ids(
|
||||
request.request_id
|
||||
)
|
||||
# The Mamba group parks its unaligned positions on the null block, so a
|
||||
# request that never failed would otherwise be restarted by this report.
|
||||
assert mamba_block_ids[0] == null_block_id
|
||||
|
||||
num_computed_tokens = request.num_computed_tokens
|
||||
_report_invalid_blocks(hybrid_scheduler, scheduler_output, {null_block_id})
|
||||
|
||||
assert request.num_computed_tokens == num_computed_tokens
|
||||
assert not hybrid_scheduler.failed_recving_kv_req_ids
|
||||
|
||||
|
||||
def _make_deepseek_v4_five_group_config(num_blocks: int) -> KVCacheConfig:
|
||||
"""Build the scheduler-side projection of V4-Flash's packed KV layout."""
|
||||
|
||||
def mla_spec() -> MLAAttentionSpec:
|
||||
return MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=4,
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
|
||||
def swa_spec(
|
||||
*, block_size: int, sliding_window: int, head_size: int
|
||||
) -> SlidingWindowMLASpec:
|
||||
return SlidingWindowMLASpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=head_size,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sliding_window,
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
|
||||
# DeepSeek-V4 packs these into five physical groups. The two SWA groups
|
||||
# have identical scheduling semantics but are split to align layer tuples.
|
||||
groups = [
|
||||
KVCacheGroupSpec(["full_mla_and_indexer"], mla_spec()),
|
||||
KVCacheGroupSpec(
|
||||
["c4_state"],
|
||||
swa_spec(block_size=4, sliding_window=8, head_size=2048),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["c128_state"],
|
||||
swa_spec(block_size=8, sliding_window=128, head_size=2048),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["swa.0"],
|
||||
swa_spec(block_size=64, sliding_window=512, head_size=512),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["swa.1"],
|
||||
swa_spec(block_size=64, sliding_window=512, head_size=512),
|
||||
),
|
||||
]
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
|
||||
|
||||
def _create_deepseek_v4_scheduler(
|
||||
kv_load_failure_policy: Literal["recompute", "fail"] = "recompute",
|
||||
) -> Scheduler:
|
||||
num_blocks = 10000
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=256,
|
||||
max_num_batched_tokens=4096,
|
||||
max_model_len=8192,
|
||||
# max_model_len only reaches SchedulerConfig; opt-125m still caps the
|
||||
# model at 2048, which is below the 9-block requests used here.
|
||||
hf_overrides={"max_position_embeddings": 8192},
|
||||
kv_load_failure_policy=kv_load_failure_policy,
|
||||
)
|
||||
return create_scheduler(
|
||||
vllm_config = create_vllm_config()
|
||||
scheduler = create_scheduler(
|
||||
vllm_config,
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_config=_make_deepseek_v4_five_group_config(num_blocks),
|
||||
hash_block_size=4,
|
||||
kv_cache_config=make_kv_cache_config(block_size=16, swa_enabled=True),
|
||||
)
|
||||
assert len(scheduler.kv_cache_config.kv_cache_groups) == 2
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def deepseek_v4_scheduler() -> Scheduler:
|
||||
return _create_deepseek_v4_scheduler()
|
||||
|
||||
|
||||
def _schedule_deepseek_v4_async_load(
|
||||
scheduler: Scheduler,
|
||||
) -> tuple[Request, SchedulerOutput]:
|
||||
num_external_tokens = 8 * scheduler.block_size
|
||||
request = create_request(num_tokens=9 * scheduler.block_size, block_size=4)
|
||||
scheduler.add_request(request)
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(
|
||||
{request.request_id: num_external_tokens}, async_load=True
|
||||
)
|
||||
)
|
||||
scheduler.connector.take_events.return_value = ()
|
||||
return request, scheduler.schedule()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failed_group_idx", range(5))
|
||||
def test_deepseek_v4_load_failure_recomputes_whole_request(
|
||||
deepseek_v4_scheduler: Scheduler, failed_group_idx: int
|
||||
):
|
||||
"""A failed load in any V4-Flash KV group must not kill EngineCore."""
|
||||
request, scheduler_output = _schedule_deepseek_v4_async_load(deepseek_v4_scheduler)
|
||||
|
||||
block_ids_per_group = deepseek_v4_scheduler.kv_cache_manager.get_block_ids(
|
||||
request.request_id
|
||||
)
|
||||
assert len(block_ids_per_group) == 5
|
||||
null_block_id = (
|
||||
deepseek_v4_scheduler.kv_cache_manager.block_pool.null_block.block_id
|
||||
)
|
||||
failed_block_id = next(
|
||||
block_id
|
||||
for block_id in block_ids_per_group[failed_group_idx]
|
||||
if block_id != null_block_id
|
||||
)
|
||||
|
||||
deepseek_v4_scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
create_model_runner_output(
|
||||
reqs=[],
|
||||
finished_recving=set(),
|
||||
invalid_block_ids={failed_block_id},
|
||||
use_eos=True,
|
||||
),
|
||||
)
|
||||
|
||||
assert request.num_computed_tokens == 0
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert deepseek_v4_scheduler.failed_recving_kv_req_ids == {request.request_id}
|
||||
|
||||
|
||||
def test_deepseek_v4_load_failure_ignores_shared_null_block(
|
||||
deepseek_v4_scheduler: Scheduler,
|
||||
):
|
||||
"""The shared null block contains no request KV and cannot fail a load."""
|
||||
request, scheduler_output = _schedule_deepseek_v4_async_load(deepseek_v4_scheduler)
|
||||
null_block_id = (
|
||||
deepseek_v4_scheduler.kv_cache_manager.block_pool.null_block.block_id
|
||||
)
|
||||
block_ids_per_group = deepseek_v4_scheduler.kv_cache_manager.get_block_ids(
|
||||
request.request_id
|
||||
)
|
||||
assert any(null_block_id in group for group in block_ids_per_group[1:])
|
||||
num_computed_tokens = request.num_computed_tokens
|
||||
|
||||
deepseek_v4_scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
create_model_runner_output(
|
||||
reqs=[],
|
||||
finished_recving=set(),
|
||||
invalid_block_ids={null_block_id},
|
||||
use_eos=True,
|
||||
),
|
||||
)
|
||||
|
||||
assert request.num_computed_tokens == num_computed_tokens
|
||||
assert not deepseek_v4_scheduler.failed_recving_kv_req_ids
|
||||
|
||||
|
||||
def test_deepseek_v4_sync_load_failure_fails_and_evicts_real_blocks():
|
||||
"""Fail policy must terminate cleanly and never evict the null block."""
|
||||
scheduler = _create_deepseek_v4_scheduler(kv_load_failure_policy="fail")
|
||||
num_external_tokens = 8 * scheduler.block_size
|
||||
request = create_request(num_tokens=9 * scheduler.block_size, block_size=4)
|
||||
scheduler.add_request(request)
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(
|
||||
{request.request_id: num_external_tokens}, async_load=False
|
||||
)
|
||||
)
|
||||
scheduler.connector.request_finished.return_value = (False, None)
|
||||
scheduler.connector.take_events.return_value = ()
|
||||
scheduler_output = scheduler.schedule()
|
||||
|
||||
block_ids_per_group = scheduler.kv_cache_manager.get_block_ids(request.request_id)
|
||||
assert len(block_ids_per_group) == 5
|
||||
null_block_id = scheduler.kv_cache_manager.block_pool.null_block.block_id
|
||||
real_block_ids = {
|
||||
block_id
|
||||
for group_block_ids in block_ids_per_group
|
||||
for block_id in group_block_ids
|
||||
if block_id != null_block_id
|
||||
}
|
||||
failed_block_id = next(
|
||||
block_id for block_id in block_ids_per_group[2] if block_id != null_block_id
|
||||
)
|
||||
|
||||
evicted_block_ids: list[set[int]] = []
|
||||
original_evict_blocks = scheduler.kv_cache_manager.evict_blocks
|
||||
|
||||
def evict_blocks_spy(block_ids: set[int]) -> None:
|
||||
evicted_block_ids.append(set(block_ids))
|
||||
original_evict_blocks(block_ids)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
scheduler.kv_cache_manager,
|
||||
"evict_blocks",
|
||||
side_effect=evict_blocks_spy,
|
||||
),
|
||||
patch.object(scheduler, "_connector_finished", return_value=(False, None)),
|
||||
):
|
||||
outputs = scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
create_model_runner_output(
|
||||
[request],
|
||||
invalid_block_ids={failed_block_id},
|
||||
use_eos=True,
|
||||
),
|
||||
)
|
||||
|
||||
assert request.status == RequestStatus.FINISHED_ERROR
|
||||
assert request.get_finished_reason() == FinishReason.ERROR
|
||||
assert request.request_id not in scheduler.requests
|
||||
assert evicted_block_ids == [real_block_ids]
|
||||
assert null_block_id not in evicted_block_ids[0]
|
||||
|
||||
engine_outputs = next(iter(outputs.values()))
|
||||
assert len(engine_outputs.outputs) == 1
|
||||
assert engine_outputs.outputs[0].request_id == request.request_id
|
||||
assert engine_outputs.outputs[0].finish_reason == FinishReason.ERROR
|
||||
with pytest.raises(RuntimeError, match="failed_recving"):
|
||||
scheduler._handle_invalid_blocks({1}, {})
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im
|
||||
MooncakeConnectorMetadata,
|
||||
MooncakeConnectorScheduler,
|
||||
MooncakeXferMetadata,
|
||||
PullReqMeta,
|
||||
SendBlockMeta,
|
||||
TransferRegion,
|
||||
)
|
||||
@@ -436,3 +437,79 @@ def test_request_finished_with_hma_groups():
|
||||
assert stored_blocks[0] == fa_blocks
|
||||
# SW: clipped to last 9 blocks (sw_size=128, block_size=16 → 8+1=9)
|
||||
assert stored_blocks[1] == sw_blocks[-9:]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Worker-side load-failure reporting (HMA vs non-HMA)
|
||||
# ---------------------------------------------------------------------------
|
||||
def _make_kv_consumer_worker(swa_enabled: bool, disable_hma: bool = False):
|
||||
block_size = 16
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_consumer",
|
||||
block_size=block_size,
|
||||
)
|
||||
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size=block_size, swa_enabled=swa_enabled
|
||||
)
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
connector = MooncakeConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
return connector.connector_worker
|
||||
|
||||
|
||||
def _make_pull_meta(d_req_id: str, local_block_ids: list[list[int]]) -> PullReqMeta:
|
||||
return PullReqMeta(
|
||||
d_req_id=d_req_id,
|
||||
transfer_id="tx-1",
|
||||
local_block_ids=local_block_ids,
|
||||
remote_engine_id="engine-0",
|
||||
remote_bootstrap_addr="127.0.0.1:1234",
|
||||
)
|
||||
|
||||
|
||||
# The tests below construct a MooncakeConnectorWorker, whose __init__
|
||||
# requires an accelerator device, so (like the worker tests in
|
||||
# test_mooncake_connector.py) they run on the GPU KV-connectors job only.
|
||||
@pytest.mark.parametrize(
|
||||
"swa_enabled,disable_hma,expected_is_hma",
|
||||
[
|
||||
(True, False, True), # SWA group present, HMA enabled
|
||||
(True, True, False), # SWA group present, but HMA disabled
|
||||
(False, False, False), # FA only, HMA not needed
|
||||
],
|
||||
)
|
||||
def test_worker_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
|
||||
"""Worker-side _is_hma_required mirrors the scheduler's derivation."""
|
||||
worker = _make_kv_consumer_worker(swa_enabled, disable_hma)
|
||||
assert worker._is_hma_required is expected_is_hma
|
||||
|
||||
|
||||
def test_worker_failed_recv_reports_request_level_failure_with_hma():
|
||||
"""With HMA, load failures report the request, not ambiguous block IDs."""
|
||||
worker = _make_kv_consumer_worker(swa_enabled=True)
|
||||
assert worker._is_hma_required
|
||||
|
||||
pull_meta = _make_pull_meta("d-req-1", [[1, 2], [3, 4]])
|
||||
worker._handle_failed_recv({"p-req-1": pull_meta}, ["p-req-1"], "boom")
|
||||
|
||||
assert worker.get_block_ids_with_load_errors() == set()
|
||||
results = worker.get_transfer_results()
|
||||
assert results.failed_recving == {"d-req-1"}
|
||||
assert results.finished_recving == {"d-req-1"}
|
||||
|
||||
|
||||
def test_worker_failed_recv_reports_block_ids_without_hma():
|
||||
"""Without HMA, load failures keep reporting block-level errors."""
|
||||
worker = _make_kv_consumer_worker(swa_enabled=False)
|
||||
assert not worker._is_hma_required
|
||||
|
||||
pull_meta = _make_pull_meta("d-req-1", [[1, 2], [3, 4]])
|
||||
worker._handle_failed_recv({"p-req-1": pull_meta}, ["p-req-1"], "boom")
|
||||
|
||||
assert worker.get_block_ids_with_load_errors() == {1, 2, 3, 4}
|
||||
results = worker.get_transfer_results()
|
||||
assert results.failed_recving == set()
|
||||
assert results.finished_recving == {"d-req-1"}
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_events import BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store import (
|
||||
connector as mooncake_store_connector,
|
||||
@@ -245,6 +246,38 @@ def test_worker_methods_delegate_to_store_worker():
|
||||
assert invalid_block_ids == {3, 4}
|
||||
|
||||
|
||||
def test_get_transfer_results_delegates_to_store_worker():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
metadata = MooncakeStoreConnectorMetadata(set(), set())
|
||||
finished_req_ids = {"req-1"}
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"connector.MooncakeStoreWorker"
|
||||
) as mock_worker_cls,
|
||||
):
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, kv_cache_config
|
||||
)
|
||||
|
||||
worker = mock_worker_cls.return_value
|
||||
transfer_results = KVConnectorTransferResults(
|
||||
finished_sending={"req-1"},
|
||||
finished_recving={"req-2"},
|
||||
failed_recving={"req-3"},
|
||||
)
|
||||
worker.get_transfer_results.return_value = transfer_results
|
||||
connector.bind_connector_metadata(metadata)
|
||||
|
||||
result = connector.get_transfer_results(finished_req_ids)
|
||||
|
||||
worker.get_transfer_results.assert_called_once_with(finished_req_ids, metadata)
|
||||
assert result is transfer_results
|
||||
|
||||
|
||||
def test_get_kv_connector_kv_cache_events_returns_none_when_disabled():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
@@ -177,6 +177,7 @@ def _make_store_recving_thread(
|
||||
*,
|
||||
tp_rank: int = 0,
|
||||
disk_offload_buffer_budget_bytes: int | None = None,
|
||||
is_hma_required: bool = False,
|
||||
) -> mooncake_store_worker.KVCacheStoreRecvingThread:
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
|
||||
|
||||
@@ -199,6 +200,7 @@ def _make_store_recving_thread(
|
||||
ready_event=threading.Event(),
|
||||
coord=coord,
|
||||
disk_offload_buffer_budget_bytes=disk_offload_buffer_budget_bytes,
|
||||
is_hma_required=is_hma_required,
|
||||
)
|
||||
thread.request_queue.task_done = MagicMock()
|
||||
return thread
|
||||
@@ -303,6 +305,7 @@ def _make_vllm_config(
|
||||
kv_role: str = "kv_both",
|
||||
pipeline_parallel_size: int = 1,
|
||||
kv_cache_layout: KVCacheLayout = KVCacheLayout.LBHNC,
|
||||
disable_hybrid_kv_cache_manager: bool = True,
|
||||
) -> SimpleNamespace:
|
||||
cache_config = SimpleNamespace(block_size=16, num_gpu_blocks=10)
|
||||
cache_config.get_resolved_kv_cache_layout = lambda: kv_cache_layout
|
||||
@@ -319,6 +322,9 @@ def _make_vllm_config(
|
||||
kv_role=kv_role, extra_config=extra_config
|
||||
),
|
||||
cache_config=cache_config,
|
||||
scheduler_config=SimpleNamespace(
|
||||
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager
|
||||
),
|
||||
kv_events_config=SimpleNamespace(enable_kv_cache_events=False),
|
||||
speculative_config=None,
|
||||
)
|
||||
@@ -2147,6 +2153,48 @@ def test_recv_thread_reports_unsplittable_key_larger_than_budget():
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {0, 1, 2}
|
||||
|
||||
|
||||
def test_recv_thread_partial_load_failure_reports_block_ids_without_hma():
|
||||
store = MagicMock()
|
||||
# First key fails, second succeeds.
|
||||
store.batch_get_into_multi_buffers.return_value = [-1, 0]
|
||||
thread = _make_store_recving_thread(store)
|
||||
|
||||
thread._handle_request(_make_load_req("req-a", [b"h0", b"h1"], token_len=32))
|
||||
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == {0}
|
||||
assert thread.get_and_clear_failed_requests() == set()
|
||||
assert thread.get_and_clear_finished_requests() == {"req-a"}
|
||||
|
||||
|
||||
def test_recv_thread_partial_load_failure_reports_request_with_hma():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [-1, 0]
|
||||
thread = _make_store_recving_thread(store, is_hma_required=True)
|
||||
|
||||
thread._handle_request(_make_load_req("req-a", [b"h0", b"h1"], token_len=32))
|
||||
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == set()
|
||||
assert thread.get_and_clear_failed_requests() == {"req-a"}
|
||||
assert thread.get_and_clear_finished_requests() == {"req-a"}
|
||||
|
||||
|
||||
def test_recv_thread_oversized_key_reports_request_with_hma():
|
||||
store = MagicMock()
|
||||
thread = _make_store_recving_thread(
|
||||
store,
|
||||
tp_rank=2,
|
||||
disk_offload_buffer_budget_bytes=_DISK_OFFLOAD_BUDGET_TOO_SMALL,
|
||||
is_hma_required=True,
|
||||
)
|
||||
|
||||
thread._handle_request(_make_load_req("req-a", [b"a0", b"a1", b"a2"], token_len=48))
|
||||
|
||||
assert store.batch_get_into_multi_buffers.call_count == 0
|
||||
assert thread.get_and_clear_block_ids_with_load_errors() == set()
|
||||
assert thread.get_and_clear_failed_requests() == {"req-a"}
|
||||
assert thread.get_and_clear_finished_requests() == {"req-a"}
|
||||
|
||||
|
||||
def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
@@ -2172,6 +2220,39 @@ def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch):
|
||||
assert [db.block_size for db in store_worker.token_dbs] == [800, 800]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("disable_hybrid_kv_cache_manager", "expected_is_hma_required"),
|
||||
[
|
||||
(True, False),
|
||||
(False, True),
|
||||
],
|
||||
)
|
||||
def test_worker_is_hma_required_from_kv_cache_groups(
|
||||
monkeypatch, disable_hybrid_kv_cache_manager, expected_is_hma_required
|
||||
):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
_install_fake_mooncake(monkeypatch, store)
|
||||
_patch_worker_runtime(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
worker.MooncakeStoreConfig,
|
||||
"load_from_config",
|
||||
staticmethod(lambda: _make_config()),
|
||||
)
|
||||
vllm_config = _make_vllm_config(
|
||||
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager
|
||||
)
|
||||
vllm_config.cache_config.block_size = 800
|
||||
vllm_config.cache_config.enable_prefix_caching = True
|
||||
vllm_config.cache_config.prefix_match_unit = None
|
||||
|
||||
store_worker = worker.MooncakeStoreWorker(
|
||||
vllm_config, _make_qsa_hybrid_kv_cache_config()
|
||||
)
|
||||
|
||||
assert store_worker._is_hma_required is expected_is_hma_required
|
||||
|
||||
|
||||
def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch):
|
||||
store = MagicMock()
|
||||
store.setup.return_value = 0
|
||||
@@ -3345,6 +3426,7 @@ def _make_bare_worker(
|
||||
worker.can_put = kv_role in ("kv_producer", "kv_both") or save_decode_cache
|
||||
worker._capacity_only = False
|
||||
worker.block_size = block_size
|
||||
worker._is_hma_required = False
|
||||
worker.tp_rank = 0
|
||||
worker.enable_kv_events = False
|
||||
worker.load_async = True
|
||||
|
||||
@@ -155,7 +155,6 @@ def create_scheduler(
|
||||
vllm_config: VllmConfig,
|
||||
num_blocks: int = 10000,
|
||||
kv_cache_config: KVCacheConfig | None = None,
|
||||
hash_block_size: int | None = None,
|
||||
) -> Scheduler | AsyncScheduler:
|
||||
"""Initialize Scheduler For Testing."""
|
||||
block_size = vllm_config.cache_config.block_size
|
||||
@@ -186,7 +185,6 @@ def create_scheduler(
|
||||
log_stats=True,
|
||||
structured_output_manager=StructuredOutputManager(vllm_config),
|
||||
block_size=block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user