From 9488318c6a40d1923614fb2d189ca93dbeb25920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Wed, 23 Sep 2026 10:15:45 +0200 Subject: [PATCH] [Mooncake] Address review nits from #56855 (#57174) Signed-off-by: NickLucche Co-authored-by: Yifan Qiao <17067717+ivanium@users.noreply.github.com> --- .../unit/test_mooncake_connector_hma.py | 56 +++++++++++++++---- .../unit/test_mooncake_store_worker.py | 41 ++++++++++---- .../v1/mooncake/mooncake_connector.py | 12 +--- .../kv_connector/v1/mooncake/store/worker.py | 8 +-- vllm/v1/core/sched/scheduler.py | 4 +- 5 files changed, 81 insertions(+), 40 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py b/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py index 37f33ce353..a7fb2be023 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector_hma.py @@ -10,6 +10,7 @@ import asyncio from unittest.mock import patch import pytest +import torch from vllm.config import set_current_vllm_config from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import ( @@ -22,6 +23,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im SendBlockMeta, TransferRegion, ) +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheConfig, + KVCacheGroupSpec, +) from .test_mooncake_connector import FakeMooncakeWrapper, patch_worker_dependencies from .utils import create_request, create_vllm_config, make_kv_cache_config @@ -440,17 +446,19 @@ def test_request_finished_with_hma_groups(): # --------------------------------------------------------------------------- # Worker-side load-failure reporting (HMA vs non-HMA) # --------------------------------------------------------------------------- -def _make_kv_consumer_worker(swa_enabled: bool, disable_hma: bool = False): +def _make_kv_consumer_worker( + swa_enabled: bool = False, kv_cache_config: KVCacheConfig | None = None +): block_size = 16 vllm_config = create_vllm_config( kv_connector="MooncakeConnector", kv_role="kv_consumer", block_size=block_size, ) - vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma - kv_cache_config = make_kv_cache_config( - block_size=block_size, swa_enabled=swa_enabled - ) + if kv_cache_config is None: + kv_cache_config = make_kv_cache_config( + block_size=block_size, swa_enabled=swa_enabled + ) with set_current_vllm_config(vllm_config), patch_worker_dependencies(): connector = MooncakeConnector( vllm_config, KVConnectorRole.WORKER, kv_cache_config @@ -472,19 +480,43 @@ def _make_pull_meta(d_req_id: str, local_block_ids: list[list[int]]) -> PullReqM # requires an accelerator device, so (like the worker tests in # test_mooncake_connector.py) they run on the GPU KV-connectors job only. @pytest.mark.parametrize( - "swa_enabled,disable_hma,expected_is_hma", + "swa_enabled,expected_is_hma", [ - (True, False, True), # SWA group present, HMA enabled - (True, True, False), # SWA group present, but HMA disabled - (False, False, False), # FA only, HMA not needed + (True, True), # multiple groups (FA + SWA) + (False, False), # single group, block-level reporting is unambiguous ], ) -def test_worker_is_hma_required(swa_enabled, disable_hma, expected_is_hma): - """Worker-side _is_hma_required mirrors the scheduler's derivation.""" - worker = _make_kv_consumer_worker(swa_enabled, disable_hma) +def test_worker_is_hma_required(swa_enabled, expected_is_hma): + """Worker-side _is_hma_required mirrors the scheduler's condition.""" + worker = _make_kv_consumer_worker(swa_enabled=swa_enabled) assert worker._is_hma_required is expected_is_hma +def test_worker_is_hma_required_multiple_full_attention_groups(): + """Two full-attention groups with different block sizes also select + request-level failure reporting, matching the scheduler.""" + kv_cache_config = KVCacheConfig( + num_blocks=100, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["layer0"], + FullAttentionSpec( + block_size=16, num_kv_heads=4, head_size=16, dtype=torch.float16 + ), + ), + KVCacheGroupSpec( + ["layer1"], + FullAttentionSpec( + block_size=32, num_kv_heads=4, head_size=16, dtype=torch.float16 + ), + ), + ], + ) + worker = _make_kv_consumer_worker(kv_cache_config=kv_cache_config) + assert worker._is_hma_required + + def test_worker_failed_recv_reports_request_level_failure_with_hma(): """With HMA, load failures report the request, not ambiguous block IDs.""" worker = _make_kv_consumer_worker(swa_enabled=True) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 6092523073..947e9f6a20 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -2222,15 +2222,40 @@ def test_worker_init_excludes_nonprefix_cache_groups(monkeypatch): assert [db.block_size for db in store_worker.token_dbs] == [800, 800] +def _make_two_full_attention_groups_kv_cache_config(): + """Two full-attention groups with different block sizes: no hybrid + layers, but block-level failure reporting is still unsupported.""" + return KVCacheConfig( + num_blocks=10, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + ["full0"], + FullAttentionSpec( + block_size=800, num_kv_heads=8, head_size=64, dtype=None + ), + ), + KVCacheGroupSpec( + ["full1"], + FullAttentionSpec( + block_size=1600, num_kv_heads=8, head_size=64, dtype=None + ), + ), + ], + ) + + @pytest.mark.parametrize( - ("disable_hybrid_kv_cache_manager", "expected_is_hma_required"), + ("make_config", "expected_is_hma_required"), [ - (True, False), - (False, True), + (lambda: _make_qsa_hybrid_kv_cache_config(), True), + (lambda: _make_two_full_attention_groups_kv_cache_config(), True), + (lambda: _make_kv_cache_config(block_size=800), False), ], + ids=["hybrid", "two-full-attention-groups", "single-group"], ) def test_worker_is_hma_required_from_kv_cache_groups( - monkeypatch, disable_hybrid_kv_cache_manager, expected_is_hma_required + monkeypatch, make_config, expected_is_hma_required ): store = MagicMock() store.setup.return_value = 0 @@ -2241,16 +2266,12 @@ def test_worker_is_hma_required_from_kv_cache_groups( "load_from_config", staticmethod(lambda: _make_config()), ) - vllm_config = _make_vllm_config( - disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager - ) + vllm_config = _make_vllm_config() vllm_config.cache_config.block_size = 800 vllm_config.cache_config.enable_prefix_caching = True vllm_config.cache_config.prefix_match_unit = None - store_worker = worker.MooncakeStoreWorker( - vllm_config, _make_qsa_hybrid_kv_cache_config() - ) + store_worker = worker.MooncakeStoreWorker(vllm_config, make_config()) assert store_worker._is_hma_required is expected_is_hma_required diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py index b22387e98b..beabf157cc 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py @@ -1051,15 +1051,9 @@ class MooncakeConnectorWorker: self.finished_recving_reqs: set[ReqId] = set() # Written from the receiver loop, drained from the worker thread. self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue() - self._is_hma_required = ( - not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager - and any( - not isinstance(g.kv_cache_spec, FullAttentionSpec) - for g in kv_cache_config.transfer_groups - ) - ) - # Block IDs are only unique within a group; with HMA the scheduler - # tracks a single merged group, so failures are reported per request. + self._is_hma_required = len(kv_cache_config.kv_cache_groups) > 1 + # Block IDs are only unique within a group, so with multiple groups + # load failures are reported per request instead. self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue() self.xfer_stats = MooncakeKVConnectorStats() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 6b6bf18ae4..baf8967bc8 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -1461,13 +1461,7 @@ class MooncakeStoreWorker: and not self.can_put ) self.cache_config = vllm_config.cache_config - self._is_hma_required = ( - not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager - and any( - not isinstance(g.kv_cache_spec, FullAttentionSpec) - for g in kv_cache_config.transfer_groups - ) - ) + self._is_hma_required = len(kv_cache_config.kv_cache_groups) > 1 self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes( kv_cache_config, vllm_config ) diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 18bbc10283..d77b63b4aa 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -3207,8 +3207,8 @@ class Scheduler(SchedulerInterface): # invalid block IDs cannot be mapped back to requests. raise RuntimeError( "A KV connector reported block-level load failures " - "(invalid_block_ids) on a layout with multiple KV cache " - "groups, where block IDs are only unique within a group. " + "(invalid_block_ids) which is not supported for models " + "with multiple KV cache groups. " "Connectors must report failed requests via " "KVConnectorTransferResults.failed_recving instead." )