From d85708f7a4362334eb58cff1fdf5265b49cac310 Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Thu, 27 Aug 2026 11:21:24 -0400 Subject: [PATCH] [1/N][KV Connector] Identify externally transferable KV cache groups (#53779) Signed-off-by: Matthew Bonanni --- tests/v1/core/test_kv_cache_utils.py | 24 +++++++++++ .../unit/test_mooncake_store_scheduler.py | 17 ++++++++ .../unit/test_nixl_connector_hma.py | 15 ++++++- tests/v1/kv_connector/unit/utils.py | 8 ++++ .../v1/mooncake/store/connector.py | 4 +- .../v1/mooncake/store/scheduler.py | 26 +++++++----- .../kv_connector/v1/mooncake/store/worker.py | 2 +- .../kv_connector/v1/nixl/base_scheduler.py | 15 ++++--- .../kv_connector/v1/nixl/base_worker.py | 12 +++--- .../kv_connector/v1/nixl/pull_worker.py | 2 +- vllm/v1/kv_cache_interface.py | 41 ++++++++++++++++++- 11 files changed, 137 insertions(+), 29 deletions(-) diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 39f18c6d4f..824bab484d 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -140,6 +140,30 @@ def new_kv_cache_spec( ) +def test_kv_cache_config_selects_only_transferable_groups(): + """Connectors must see a stable projection of transfer-eligible groups.""" + groups = [ + KVCacheGroupSpec(["layer.0"], new_kv_cache_spec()), + KVCacheGroupSpec(["layer.1"], new_kv_cache_spec(), enable_kv_transfer=False), + KVCacheGroupSpec(["layer.2"], new_kv_cache_spec()), + ] + config = KVCacheConfig( + num_blocks=1, + kv_cache_tensors=[], + kv_cache_groups=groups, + ) + + assert config.transfer_group_ids == (0, 2) + assert config.transfer_groups == (groups[0], groups[2]) + assert config.transfer_group_index_by_layer == {"layer.0": 0, "layer.2": 1} + first_blocks = [1, 2] + third_blocks = [4] + assert config.select_transfer_block_ids((first_blocks, [3], third_blocks)) == ( + first_blocks, + third_blocks, + ) + + def new_sliding_window_spec( block_size=16, num_kv_heads=2, diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py index 1f02d18900..b1ce5aeef9 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py @@ -33,6 +33,9 @@ def _make_bare_scheduler( scheduler._block_size = 16 scheduler._hash_block_size = hash_block_size scheduler.enable_partial_hash_hits = enable_partial_hash_hits + scheduler.kv_cache_config = SimpleNamespace( + select_transfer_block_ids=lambda block_ids: tuple(block_ids) + ) scheduler.load_specs = {} scheduler._unfinished_request_ids = {"req-0"} scheduler._unfinished_requests = {} @@ -179,6 +182,20 @@ def _add_unfinished_request( ) +def test_update_state_excludes_nontransfer_groups(): + """Store metadata must match the worker's registered cache groups.""" + scheduler = _make_bare_scheduler() + scheduler.kv_cache_config = SimpleNamespace( + select_transfer_block_ids=lambda block_ids: (block_ids[0],) + ) + request = SimpleNamespace(request_id="req-1") + blocks = SimpleNamespace(get_block_ids=lambda: ([1, 2], [9])) + + scheduler.update_state_after_alloc(request, blocks, num_external_tokens=32) + + assert scheduler._unfinished_requests["req-1"][1] == ([1, 2],) + + def _setup_decode_request( *, kv_role: str = "kv_consumer", diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 11d5328ba1..f5bf9e9be1 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -854,7 +854,10 @@ def test_post_process_zeroes_untransferred_tail(): worker.device_kv_caches = {"attn.0": attn_cache, "mamba.0": mamba_cache} fa_group = MagicMock(layer_names=["attn.0"]) ssm_group = MagicMock(layer_names=["mamba.0"]) - worker.kv_cache_config = MagicMock(kv_cache_groups=[fa_group, ssm_group]) + worker.kv_cache_config = MagicMock( + kv_cache_groups=[fa_group, ssm_group], + transfer_groups=[fa_group, ssm_group], + ) # The cached property filters mamba layers out of the permuted caches. attn_caches = NixlConnectorWorker._attention_kv_caches.func(worker) assert len(attn_caches) == 1 and attn_caches[0] is attn_cache @@ -1477,6 +1480,16 @@ def test_exchange_clipped_blocks_ssm_positional_states(): assert clipped == ([1, 2, 3], [0, 5, 6, 7]) +@pytest.mark.cpu_test +def test_exchange_clipped_blocks_excludes_nontransfer_groups(): + """Scheduler block tables must match the worker's registered regions.""" + sched = make_nixl_scheduler() + sched.kv_cache_config = make_kv_cache_config(block_size=16, swa_enabled=True) + sched.kv_cache_config.kv_cache_groups[1].enable_kv_transfer = False + + assert sched.get_exchange_clipped_blocks(([1, 2], [9])) == ([1, 2],) + + # ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ───────────────────── diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index 1bc09995a3..b815fbec96 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -524,6 +524,10 @@ def make_nixl_scheduler( sched = object.__new__(NixlConnectorScheduler) sched._has_mamba = has_mamba sched._is_hma_required = is_hma_required + sched.kv_cache_config = make_kv_cache_config( + block_size=16, + mamba_enabled=has_mamba, + ) if heartbeat: sched._heartbeat_by_engine = {} @@ -582,6 +586,10 @@ def make_nixl_push_scheduler( sched.side_channel_port = 5600 sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled sched._has_mamba = has_mamba + sched.kv_cache_config = make_kv_cache_config( + block_size=16, + mamba_enabled=has_mamba, + ) # vllm_config is consulted for parallel_config.tensor_parallel_size. vllm_config = MagicMock() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py index 77adc2d1bc..955fddc715 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py @@ -97,7 +97,7 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): unsupported: list[str] = [] cache_block_size = vllm_config.cache_config.block_size - for g_idx, g in enumerate(kv_cache_config.kv_cache_groups): + for g_idx, g in enumerate(kv_cache_config.transfer_groups): spec = g.kv_cache_spec if isinstance(spec, CrossAttentionSpec): unsupported.append(f"group {g_idx}: CrossAttentionSpec") @@ -110,7 +110,7 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): ) pcp = vllm_config.parallel_config.prefill_context_parallel_size dcp = vllm_config.parallel_config.decode_context_parallel_size - if len(kv_cache_config.kv_cache_groups) > 1 and pcp * dcp > 1: + if len(kv_cache_config.transfer_groups) > 1 and pcp * dcp > 1: unsupported.append( f"PCP/DCP > 1 (pcp={pcp}, dcp={dcp}) with hybrid attention" ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py index c26e25fc5a..3abf7da4f0 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py @@ -69,6 +69,7 @@ class MooncakeStoreScheduler: kv_event_config and kv_event_config.enable_kv_cache_events ) self.client = LookupKeyClient(vllm_config) + self.kv_cache_config = kv_cache_config # Align with the engine's own scheduler_block_size and hash_block_size. self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes( @@ -157,7 +158,9 @@ class MooncakeStoreScheduler: """Update state after block allocation.""" local_block_ids: tuple[list[int], ...] = () if num_external_tokens > 0: - local_block_ids = blocks.get_block_ids() + local_block_ids = self.kv_cache_config.select_transfer_block_ids( + blocks.get_block_ids() + ) self._unfinished_requests[request.request_id] = (request, local_block_ids) self._unfinished_request_ids.add(request.request_id) @@ -220,12 +223,12 @@ class MooncakeStoreScheduler: request_tuple = self._unfinished_requests.get(request.req_id) request_real = request_tuple[0] # type: ignore[index] - if isinstance(request.block_ids, tuple): - # Multi-group: preserve per-group structure. - unfolded_block_ids = tuple(b.copy() for b in request.block_ids) - else: - # Single-group legacy: list[int] -> 1-tuple. - unfolded_block_ids = (request.block_ids.copy(),) + unfolded_block_ids = tuple( + blocks.copy() + for blocks in self.kv_cache_config.select_transfer_block_ids( + request.block_ids + ) + ) prefill_tokens = _new_req_prefill_tokens(request) request_tracker = RequestTracker( @@ -259,16 +262,17 @@ class MooncakeStoreScheduler: if can_process_cached: for i, req_id in enumerate(cached_reqs.req_ids): new_block_ids = cached_reqs.new_block_ids[i] + if new_block_ids: + new_block_ids = self.kv_cache_config.select_transfer_block_ids( + new_block_ids + ) req_meta = None if req_id in cached_reqs.resumed_req_ids: # Resumed after preemption if not new_block_ids: continue - if isinstance(new_block_ids, tuple): - new_block_ids = tuple(b.copy() for b in new_block_ids) - else: - new_block_ids = (new_block_ids.copy(),) + new_block_ids = tuple(b.copy() for b in new_block_ids) load_spec = self.load_specs.pop(req_id, None) request_tuple = self._unfinished_requests.get(req_id) request_real = request_tuple[0] # type: ignore[index] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 4b73614ae5..b48e676835 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -1497,7 +1497,7 @@ class MooncakeStoreWorker: # Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to # self.block_size (= scheduler_block_size) so the coordinator's # ``block_size % hash_block_size == 0`` invariant holds. - groups = list(kv_cache_config.kv_cache_groups) + groups = list(kv_cache_config.transfer_groups) if len(groups) == 1 and groups[0].kv_cache_spec.block_size != self.block_size: g = groups[0] groups = [ diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py index 9b623b53c7..eb2bf76d95 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py @@ -90,12 +90,12 @@ class NixlBaseConnectorScheduler: # Also handle unlikely SW-only model case instead of checking num_groups>1. and any( not isinstance(g.kv_cache_spec, FullAttentionSpec) - for g in kv_cache_config.kv_cache_groups + for g in kv_cache_config.transfer_groups ) ) self._has_mamba = any( isinstance(g.kv_cache_spec, MambaSpec) - for g in kv_cache_config.kv_cache_groups + for g in kv_cache_config.transfer_groups ) logger.info("Initializing NIXL Scheduler %s", engine_id) @@ -131,7 +131,7 @@ class NixlBaseConnectorScheduler: (g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size) if isinstance(g.kv_cache_spec, SlidingWindowSpec) else (0, self.block_size) - for g in kv_cache_config.kv_cache_groups + for g in kv_cache_config.transfer_groups ] # cdiv(n_tokens, block_size) gives blocks/window; add 1 to conservatively # account for boundary overlap eg window isn't fully aligned with blocks. @@ -146,7 +146,7 @@ class NixlBaseConnectorScheduler: g.kv_cache_spec.num_speculative_blocks if isinstance(g.kv_cache_spec, MambaSpec) else None - for g in kv_cache_config.kv_cache_groups + for g in kv_cache_config.transfer_groups ] # Only "all" mode keeps a state per block position; the other modes # keep a single running state in the last non-speculative slot. @@ -260,8 +260,11 @@ class NixlBaseConnectorScheduler: for per-step partial lists (host-buffer save), where the SSM strip does not apply. """ - if len(block_ids) == 0 or not self._is_hma_required: - # No blocks to clip eg Full prefix cache hit or not a hybrid model. + if len(block_ids) == 0: + # No blocks to clip, e.g. a full prefix cache hit. + return block_ids + block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids) + if not self._is_hma_required: return block_ids # NOTE (NickLucche) This logic is currently handled at the connector level # because offloading connectors might want to receive the whole sequence even diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index e7267090c8..c8166b2e6a 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -300,13 +300,13 @@ class NixlBaseConnectorWorker: not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager and any( not isinstance(g.kv_cache_spec, FullAttentionSpec) - for g in kv_cache_config.kv_cache_groups + for g in kv_cache_config.transfer_groups ) ) self.kv_cache_config = kv_cache_config self._layer_specs = { layer: group.kv_cache_spec - for group in kv_cache_config.kv_cache_groups + for group in kv_cache_config.transfer_groups for layer in group.layer_names } @@ -318,7 +318,7 @@ class NixlBaseConnectorWorker: self._conv_decomp: MambaConvSplitInfo | None = None self._has_mamba = any( isinstance(g.kv_cache_spec, MambaSpec) - for g in kv_cache_config.kv_cache_groups + for g in kv_cache_config.transfer_groups ) if self._has_mamba: assert self._is_hma_required @@ -554,7 +554,7 @@ class NixlBaseConnectorWorker: # Unwrap UniformTypeKVCacheSpecs to get the representative spec type self._group_spec_types = tuple( get_representative_spec_type(g.kv_cache_spec) - for g in self.kv_cache_config.kv_cache_groups + for g in self.kv_cache_config.transfer_groups ) # Per-region MLA flag, 1:1 with block_len_per_layer. True -> REPLICATE @@ -1906,7 +1906,7 @@ class NixlBaseConnectorWorker: ) mamba_layers = { name - for g, group in enumerate(self.kv_cache_config.kv_cache_groups) + for g, group in enumerate(self.kv_cache_config.transfer_groups) if _is_ssm_spec(self._group_spec_types[g]) for name in group.layer_names } @@ -2361,7 +2361,7 @@ class NixlBaseConnectorWorker: return block_ids block_arange = np.arange(0, ratio).reshape(1, -1) # Mamba blocks have no logical<>physical discrepancy (block-size=1) - group_specs = self.kv_cache_config.kv_cache_groups + group_specs = self.kv_cache_config.transfer_groups physical_block_ids = [] for i, group in enumerate(block_ids): spec = group_specs[i].kv_cache_spec diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index ea04702e1b..a09fd684e3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -290,7 +290,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): assert ( len(remote_block_ids) == len(local_block_ids) - == len(self.kv_cache_config.kv_cache_groups) + == len(self.kv_cache_config.transfer_groups) ) local_block_ids, remote_block_ids = self._apply_prefix_caching( decode_block_ids=local_block_ids, diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 55c9e806a5..d5f7cff7c1 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -5,10 +5,11 @@ from __future__ import annotations import copy from collections import Counter -from collections.abc import Collection +from collections.abc import Collection, Sequence from dataclasses import dataclass, fields, replace from enum import Enum, IntEnum from fractions import Fraction +from functools import cached_property from math import prod from typing import TYPE_CHECKING, TypeVar @@ -1143,6 +1144,8 @@ class KVCacheGroupSpec: kv_cache_spec: KVCacheSpec # Whether this group contains EAGLE/MTP draft attention layers. is_eagle_group: bool = False + # Whether this group is part of the externally transferable KV state. + enable_kv_transfer: bool = True @dataclass @@ -1168,6 +1171,42 @@ class KVCacheConfig: kv_cache_layout: str | None = None """The KV cache layout resolved by the engine core, adopted by all workers.""" + @cached_property + def transfer_group_ids(self) -> tuple[int, ...]: + """IDs of cache groups that participate in external KV transfer.""" + return tuple( + group_id + for group_id, group in enumerate(self.kv_cache_groups) + if group.enable_kv_transfer + ) + + @cached_property + def transfer_groups(self) -> tuple[KVCacheGroupSpec, ...]: + """Cache groups that participate in external KV transfer.""" + return tuple( + self.kv_cache_groups[group_id] for group_id in self.transfer_group_ids + ) + + @cached_property + def transfer_group_index_by_layer(self) -> dict[str, int]: + """Transfer-group tuple index for each participating layer.""" + return { + layer_name: group_index + for group_index, group in enumerate(self.transfer_groups) + for layer_name in group.layer_names + } + + def select_transfer_block_ids( + self, block_ids: Sequence[list[int]] + ) -> tuple[list[int], ...]: + """Select block IDs for externally transferable cache groups.""" + if len(block_ids) != len(self.kv_cache_groups): + raise ValueError( + f"Expected {len(self.kv_cache_groups)} KV cache groups, " + f"got {len(block_ids)}." + ) + return tuple(block_ids[group_id] for group_id in self.transfer_group_ids) + @property def has_mamba_layers(self) -> bool: return any(isinstance(g.kv_cache_spec, MambaSpec) for g in self.kv_cache_groups)