[Mooncake] Address review nits from #56855 (#57174)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: Yifan Qiao <17067717+ivanium@users.noreply.github.com>
This commit is contained in:
Nicolò Lucchesi
2026-09-23 10:15:45 +02:00
committed by GitHub
co-authored by Yifan Qiao
parent 153242a314
commit 9488318c6a
5 changed files with 81 additions and 40 deletions
@@ -10,6 +10,7 @@ import asyncio
from unittest.mock import patch
import pytest
import torch
from vllm.config import set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
@@ -22,6 +23,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im
SendBlockMeta,
TransferRegion,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
)
from .test_mooncake_connector import FakeMooncakeWrapper, patch_worker_dependencies
from .utils import create_request, create_vllm_config, make_kv_cache_config
@@ -440,17 +446,19 @@ def test_request_finished_with_hma_groups():
# ---------------------------------------------------------------------------
# Worker-side load-failure reporting (HMA vs non-HMA)
# ---------------------------------------------------------------------------
def _make_kv_consumer_worker(swa_enabled: bool, disable_hma: bool = False):
def _make_kv_consumer_worker(
swa_enabled: bool = False, kv_cache_config: KVCacheConfig | None = None
):
block_size = 16
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_consumer",
block_size=block_size,
)
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma
kv_cache_config = make_kv_cache_config(
block_size=block_size, swa_enabled=swa_enabled
)
if kv_cache_config is None:
kv_cache_config = make_kv_cache_config(
block_size=block_size, swa_enabled=swa_enabled
)
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
connector = MooncakeConnector(
vllm_config, KVConnectorRole.WORKER, kv_cache_config
@@ -472,19 +480,43 @@ def _make_pull_meta(d_req_id: str, local_block_ids: list[list[int]]) -> PullReqM
# requires an accelerator device, so (like the worker tests in
# test_mooncake_connector.py) they run on the GPU KV-connectors job only.
@pytest.mark.parametrize(
"swa_enabled,disable_hma,expected_is_hma",
"swa_enabled,expected_is_hma",
[
(True, False, True), # SWA group present, HMA enabled
(True, True, False), # SWA group present, but HMA disabled
(False, False, False), # FA only, HMA not needed
(True, True), # multiple groups (FA + SWA)
(False, False), # single group, block-level reporting is unambiguous
],
)
def test_worker_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
"""Worker-side _is_hma_required mirrors the scheduler's derivation."""
worker = _make_kv_consumer_worker(swa_enabled, disable_hma)
def test_worker_is_hma_required(swa_enabled, expected_is_hma):
"""Worker-side _is_hma_required mirrors the scheduler's condition."""
worker = _make_kv_consumer_worker(swa_enabled=swa_enabled)
assert worker._is_hma_required is expected_is_hma
def test_worker_is_hma_required_multiple_full_attention_groups():
"""Two full-attention groups with different block sizes also select
request-level failure reporting, matching the scheduler."""
kv_cache_config = KVCacheConfig(
num_blocks=100,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=16, dtype=torch.float16
),
),
KVCacheGroupSpec(
["layer1"],
FullAttentionSpec(
block_size=32, num_kv_heads=4, head_size=16, dtype=torch.float16
),
),
],
)
worker = _make_kv_consumer_worker(kv_cache_config=kv_cache_config)
assert worker._is_hma_required
def test_worker_failed_recv_reports_request_level_failure_with_hma():
"""With HMA, load failures report the request, not ambiguous block IDs."""
worker = _make_kv_consumer_worker(swa_enabled=True)
@@ -2222,15 +2222,40 @@ def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch):
assert [db.block_size for db in store_worker.token_dbs] == [800, 800]
def _make_two_full_attention_groups_kv_cache_config():
"""Two full-attention groups with different block sizes: no hybrid
layers, but block-level failure reporting is still unsupported."""
return KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full0"],
FullAttentionSpec(
block_size=800, num_kv_heads=8, head_size=64, dtype=None
),
),
KVCacheGroupSpec(
["full1"],
FullAttentionSpec(
block_size=1600, num_kv_heads=8, head_size=64, dtype=None
),
),
],
)
@pytest.mark.parametrize(
("disable_hybrid_kv_cache_manager", "expected_is_hma_required"),
("make_config", "expected_is_hma_required"),
[
(True, False),
(False, True),
(lambda: _make_qsa_hybrid_kv_cache_config(), True),
(lambda: _make_two_full_attention_groups_kv_cache_config(), True),
(lambda: _make_kv_cache_config(block_size=800), False),
],
ids=["hybrid", "two-full-attention-groups", "single-group"],
)
def test_worker_is_hma_required_from_kv_cache_groups(
monkeypatch, disable_hybrid_kv_cache_manager, expected_is_hma_required
monkeypatch, make_config, expected_is_hma_required
):
store = MagicMock()
store.setup.return_value = 0
@@ -2241,16 +2266,12 @@ def test_worker_is_hma_required_from_kv_cache_groups(
"load_from_config",
staticmethod(lambda: _make_config()),
)
vllm_config = _make_vllm_config(
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager
)
vllm_config = _make_vllm_config()
vllm_config.cache_config.block_size = 800
vllm_config.cache_config.enable_prefix_caching = True
vllm_config.cache_config.prefix_match_unit = None
store_worker = worker.MooncakeStoreWorker(
vllm_config, _make_qsa_hybrid_kv_cache_config()
)
store_worker = worker.MooncakeStoreWorker(vllm_config, make_config())
assert store_worker._is_hma_required is expected_is_hma_required
@@ -1051,15 +1051,9 @@ class MooncakeConnectorWorker:
self.finished_recving_reqs: set[ReqId] = set()
# Written from the receiver loop, drained from the worker thread.
self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue()
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.transfer_groups
)
)
# Block IDs are only unique within a group; with HMA the scheduler
# tracks a single merged group, so failures are reported per request.
self._is_hma_required = len(kv_cache_config.kv_cache_groups) > 1
# Block IDs are only unique within a group, so with multiple groups
# load failures are reported per request instead.
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
self.xfer_stats = MooncakeKVConnectorStats()
@@ -1461,13 +1461,7 @@ class MooncakeStoreWorker:
and not self.can_put
)
self.cache_config = vllm_config.cache_config
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.transfer_groups
)
)
self._is_hma_required = len(kv_cache_config.kv_cache_groups) > 1
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
+2 -2
View File
@@ -3207,8 +3207,8 @@ class Scheduler(SchedulerInterface):
# invalid block IDs cannot be mapped back to requests.
raise RuntimeError(
"A KV connector reported block-level load failures "
"(invalid_block_ids) on a layout with multiple KV cache "
"groups, where block IDs are only unique within a group. "
"(invalid_block_ids) which is not supported for models "
"with multiple KV cache groups. "
"Connectors must report failed requests via "
"KVConnectorTransferResults.failed_recving instead."
)