[1/N][KV Connector] Identify externally transferable KV cache groups (#53779)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Matthew Bonanni
2026-08-27 11:21:24 -04:00
committed by GitHub
parent c7b0467c25
commit d85708f7a4
11 changed files with 137 additions and 29 deletions
+24
View File
@@ -140,6 +140,30 @@ def new_kv_cache_spec(
)
def test_kv_cache_config_selects_only_transferable_groups():
"""Connectors must see a stable projection of transfer-eligible groups."""
groups = [
KVCacheGroupSpec(["layer.0"], new_kv_cache_spec()),
KVCacheGroupSpec(["layer.1"], new_kv_cache_spec(), enable_kv_transfer=False),
KVCacheGroupSpec(["layer.2"], new_kv_cache_spec()),
]
config = KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=groups,
)
assert config.transfer_group_ids == (0, 2)
assert config.transfer_groups == (groups[0], groups[2])
assert config.transfer_group_index_by_layer == {"layer.0": 0, "layer.2": 1}
first_blocks = [1, 2]
third_blocks = [4]
assert config.select_transfer_block_ids((first_blocks, [3], third_blocks)) == (
first_blocks,
third_blocks,
)
def new_sliding_window_spec(
block_size=16,
num_kv_heads=2,
@@ -33,6 +33,9 @@ def _make_bare_scheduler(
scheduler._block_size = 16
scheduler._hash_block_size = hash_block_size
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
scheduler.kv_cache_config = SimpleNamespace(
select_transfer_block_ids=lambda block_ids: tuple(block_ids)
)
scheduler.load_specs = {}
scheduler._unfinished_request_ids = {"req-0"}
scheduler._unfinished_requests = {}
@@ -179,6 +182,20 @@ def _add_unfinished_request(
)
def test_update_state_excludes_nontransfer_groups():
"""Store metadata must match the worker's registered cache groups."""
scheduler = _make_bare_scheduler()
scheduler.kv_cache_config = SimpleNamespace(
select_transfer_block_ids=lambda block_ids: (block_ids[0],)
)
request = SimpleNamespace(request_id="req-1")
blocks = SimpleNamespace(get_block_ids=lambda: ([1, 2], [9]))
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=32)
assert scheduler._unfinished_requests["req-1"][1] == ([1, 2],)
def _setup_decode_request(
*,
kv_role: str = "kv_consumer",
@@ -854,7 +854,10 @@ def test_post_process_zeroes_untransferred_tail():
worker.device_kv_caches = {"attn.0": attn_cache, "mamba.0": mamba_cache}
fa_group = MagicMock(layer_names=["attn.0"])
ssm_group = MagicMock(layer_names=["mamba.0"])
worker.kv_cache_config = MagicMock(kv_cache_groups=[fa_group, ssm_group])
worker.kv_cache_config = MagicMock(
kv_cache_groups=[fa_group, ssm_group],
transfer_groups=[fa_group, ssm_group],
)
# The cached property filters mamba layers out of the permuted caches.
attn_caches = NixlConnectorWorker._attention_kv_caches.func(worker)
assert len(attn_caches) == 1 and attn_caches[0] is attn_cache
@@ -1477,6 +1480,16 @@ def test_exchange_clipped_blocks_ssm_positional_states():
assert clipped == ([1, 2, 3], [0, 5, 6, 7])
@pytest.mark.cpu_test
def test_exchange_clipped_blocks_excludes_nontransfer_groups():
"""Scheduler block tables must match the worker's registered regions."""
sched = make_nixl_scheduler()
sched.kv_cache_config = make_kv_cache_config(block_size=16, swa_enabled=True)
sched.kv_cache_config.kv_cache_groups[1].enable_kv_transfer = False
assert sched.get_exchange_clipped_blocks(([1, 2], [9])) == ([1, 2],)
# ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ─────────────────────
+8
View File
@@ -524,6 +524,10 @@ def make_nixl_scheduler(
sched = object.__new__(NixlConnectorScheduler)
sched._has_mamba = has_mamba
sched._is_hma_required = is_hma_required
sched.kv_cache_config = make_kv_cache_config(
block_size=16,
mamba_enabled=has_mamba,
)
if heartbeat:
sched._heartbeat_by_engine = {}
@@ -582,6 +586,10 @@ def make_nixl_push_scheduler(
sched.side_channel_port = 5600
sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled
sched._has_mamba = has_mamba
sched.kv_cache_config = make_kv_cache_config(
block_size=16,
mamba_enabled=has_mamba,
)
# vllm_config is consulted for parallel_config.tensor_parallel_size.
vllm_config = MagicMock()
@@ -97,7 +97,7 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
unsupported: list[str] = []
cache_block_size = vllm_config.cache_config.block_size
for g_idx, g in enumerate(kv_cache_config.kv_cache_groups):
for g_idx, g in enumerate(kv_cache_config.transfer_groups):
spec = g.kv_cache_spec
if isinstance(spec, CrossAttentionSpec):
unsupported.append(f"group {g_idx}: CrossAttentionSpec")
@@ -110,7 +110,7 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
)
pcp = vllm_config.parallel_config.prefill_context_parallel_size
dcp = vllm_config.parallel_config.decode_context_parallel_size
if len(kv_cache_config.kv_cache_groups) > 1 and pcp * dcp > 1:
if len(kv_cache_config.transfer_groups) > 1 and pcp * dcp > 1:
unsupported.append(
f"PCP/DCP > 1 (pcp={pcp}, dcp={dcp}) with hybrid attention"
)
@@ -69,6 +69,7 @@ class MooncakeStoreScheduler:
kv_event_config and kv_event_config.enable_kv_cache_events
)
self.client = LookupKeyClient(vllm_config)
self.kv_cache_config = kv_cache_config
# Align with the engine's own scheduler_block_size and hash_block_size.
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
@@ -157,7 +158,9 @@ class MooncakeStoreScheduler:
"""Update state after block allocation."""
local_block_ids: tuple[list[int], ...] = ()
if num_external_tokens > 0:
local_block_ids = blocks.get_block_ids()
local_block_ids = self.kv_cache_config.select_transfer_block_ids(
blocks.get_block_ids()
)
self._unfinished_requests[request.request_id] = (request, local_block_ids)
self._unfinished_request_ids.add(request.request_id)
@@ -220,12 +223,12 @@ class MooncakeStoreScheduler:
request_tuple = self._unfinished_requests.get(request.req_id)
request_real = request_tuple[0] # type: ignore[index]
if isinstance(request.block_ids, tuple):
# Multi-group: preserve per-group structure.
unfolded_block_ids = tuple(b.copy() for b in request.block_ids)
else:
# Single-group legacy: list[int] -> 1-tuple.
unfolded_block_ids = (request.block_ids.copy(),)
unfolded_block_ids = tuple(
blocks.copy()
for blocks in self.kv_cache_config.select_transfer_block_ids(
request.block_ids
)
)
prefill_tokens = _new_req_prefill_tokens(request)
request_tracker = RequestTracker(
@@ -259,16 +262,17 @@ class MooncakeStoreScheduler:
if can_process_cached:
for i, req_id in enumerate(cached_reqs.req_ids):
new_block_ids = cached_reqs.new_block_ids[i]
if new_block_ids:
new_block_ids = self.kv_cache_config.select_transfer_block_ids(
new_block_ids
)
req_meta = None
if req_id in cached_reqs.resumed_req_ids:
# Resumed after preemption
if not new_block_ids:
continue
if isinstance(new_block_ids, tuple):
new_block_ids = tuple(b.copy() for b in new_block_ids)
else:
new_block_ids = (new_block_ids.copy(),)
new_block_ids = tuple(b.copy() for b in new_block_ids)
load_spec = self.load_specs.pop(req_id, None)
request_tuple = self._unfinished_requests.get(req_id)
request_real = request_tuple[0] # type: ignore[index]
@@ -1497,7 +1497,7 @@ class MooncakeStoreWorker:
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
# self.block_size (= scheduler_block_size) so the coordinator's
# ``block_size % hash_block_size == 0`` invariant holds.
groups = list(kv_cache_config.kv_cache_groups)
groups = list(kv_cache_config.transfer_groups)
if len(groups) == 1 and groups[0].kv_cache_spec.block_size != self.block_size:
g = groups[0]
groups = [
@@ -90,12 +90,12 @@ class NixlBaseConnectorScheduler:
# Also handle unlikely SW-only model case instead of checking num_groups>1.
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
)
)
self._has_mamba = any(
isinstance(g.kv_cache_spec, MambaSpec)
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
)
logger.info("Initializing NIXL Scheduler %s", engine_id)
@@ -131,7 +131,7 @@ class NixlBaseConnectorScheduler:
(g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size)
if isinstance(g.kv_cache_spec, SlidingWindowSpec)
else (0, self.block_size)
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
]
# cdiv(n_tokens, block_size) gives blocks/window; add 1 to conservatively
# account for boundary overlap eg window isn't fully aligned with blocks.
@@ -146,7 +146,7 @@ class NixlBaseConnectorScheduler:
g.kv_cache_spec.num_speculative_blocks
if isinstance(g.kv_cache_spec, MambaSpec)
else None
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
]
# Only "all" mode keeps a state per block position; the other modes
# keep a single running state in the last non-speculative slot.
@@ -260,8 +260,11 @@ class NixlBaseConnectorScheduler:
for per-step partial lists (host-buffer save), where the SSM strip
does not apply.
"""
if len(block_ids) == 0 or not self._is_hma_required:
# No blocks to clip eg Full prefix cache hit or not a hybrid model.
if len(block_ids) == 0:
# No blocks to clip, e.g. a full prefix cache hit.
return block_ids
block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids)
if not self._is_hma_required:
return block_ids
# NOTE (NickLucche) This logic is currently handled at the connector level
# because offloading connectors might want to receive the whole sequence even
@@ -300,13 +300,13 @@ class NixlBaseConnectorWorker:
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
)
)
self.kv_cache_config = kv_cache_config
self._layer_specs = {
layer: group.kv_cache_spec
for group in kv_cache_config.kv_cache_groups
for group in kv_cache_config.transfer_groups
for layer in group.layer_names
}
@@ -318,7 +318,7 @@ class NixlBaseConnectorWorker:
self._conv_decomp: MambaConvSplitInfo | None = None
self._has_mamba = any(
isinstance(g.kv_cache_spec, MambaSpec)
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
)
if self._has_mamba:
assert self._is_hma_required
@@ -554,7 +554,7 @@ class NixlBaseConnectorWorker:
# Unwrap UniformTypeKVCacheSpecs to get the representative spec type
self._group_spec_types = tuple(
get_representative_spec_type(g.kv_cache_spec)
for g in self.kv_cache_config.kv_cache_groups
for g in self.kv_cache_config.transfer_groups
)
# Per-region MLA flag, 1:1 with block_len_per_layer. True -> REPLICATE
@@ -1906,7 +1906,7 @@ class NixlBaseConnectorWorker:
)
mamba_layers = {
name
for g, group in enumerate(self.kv_cache_config.kv_cache_groups)
for g, group in enumerate(self.kv_cache_config.transfer_groups)
if _is_ssm_spec(self._group_spec_types[g])
for name in group.layer_names
}
@@ -2361,7 +2361,7 @@ class NixlBaseConnectorWorker:
return block_ids
block_arange = np.arange(0, ratio).reshape(1, -1)
# Mamba blocks have no logical<>physical discrepancy (block-size=1)
group_specs = self.kv_cache_config.kv_cache_groups
group_specs = self.kv_cache_config.transfer_groups
physical_block_ids = []
for i, group in enumerate(block_ids):
spec = group_specs[i].kv_cache_spec
@@ -290,7 +290,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
assert (
len(remote_block_ids)
== len(local_block_ids)
== len(self.kv_cache_config.kv_cache_groups)
== len(self.kv_cache_config.transfer_groups)
)
local_block_ids, remote_block_ids = self._apply_prefix_caching(
decode_block_ids=local_block_ids,
+40 -1
View File
@@ -5,10 +5,11 @@ from __future__ import annotations
import copy
from collections import Counter
from collections.abc import Collection
from collections.abc import Collection, Sequence
from dataclasses import dataclass, fields, replace
from enum import Enum, IntEnum
from fractions import Fraction
from functools import cached_property
from math import prod
from typing import TYPE_CHECKING, TypeVar
@@ -1143,6 +1144,8 @@ class KVCacheGroupSpec:
kv_cache_spec: KVCacheSpec
# Whether this group contains EAGLE/MTP draft attention layers.
is_eagle_group: bool = False
# Whether this group is part of the externally transferable KV state.
enable_kv_transfer: bool = True
@dataclass
@@ -1168,6 +1171,42 @@ class KVCacheConfig:
kv_cache_layout: str | None = None
"""The KV cache layout resolved by the engine core, adopted by all workers."""
@cached_property
def transfer_group_ids(self) -> tuple[int, ...]:
"""IDs of cache groups that participate in external KV transfer."""
return tuple(
group_id
for group_id, group in enumerate(self.kv_cache_groups)
if group.enable_kv_transfer
)
@cached_property
def transfer_groups(self) -> tuple[KVCacheGroupSpec, ...]:
"""Cache groups that participate in external KV transfer."""
return tuple(
self.kv_cache_groups[group_id] for group_id in self.transfer_group_ids
)
@cached_property
def transfer_group_index_by_layer(self) -> dict[str, int]:
"""Transfer-group tuple index for each participating layer."""
return {
layer_name: group_index
for group_index, group in enumerate(self.transfer_groups)
for layer_name in group.layer_names
}
def select_transfer_block_ids(
self, block_ids: Sequence[list[int]]
) -> tuple[list[int], ...]:
"""Select block IDs for externally transferable cache groups."""
if len(block_ids) != len(self.kv_cache_groups):
raise ValueError(
f"Expected {len(self.kv_cache_groups)} KV cache groups, "
f"got {len(block_ids)}."
)
return tuple(block_ids[group_id] for group_id in self.transfer_group_ids)
@property
def has_mamba_layers(self) -> bool:
return any(isinstance(g.kv_cache_spec, MambaSpec) for g in self.kv_cache_groups)