[1/N][KV Connector] Identify externally transferable KV cache groups (#53779)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
@@ -140,6 +140,30 @@ def new_kv_cache_spec(
|
||||
)
|
||||
|
||||
|
||||
def test_kv_cache_config_selects_only_transferable_groups():
|
||||
"""Connectors must see a stable projection of transfer-eligible groups."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["layer.0"], new_kv_cache_spec()),
|
||||
KVCacheGroupSpec(["layer.1"], new_kv_cache_spec(), enable_kv_transfer=False),
|
||||
KVCacheGroupSpec(["layer.2"], new_kv_cache_spec()),
|
||||
]
|
||||
config = KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
|
||||
assert config.transfer_group_ids == (0, 2)
|
||||
assert config.transfer_groups == (groups[0], groups[2])
|
||||
assert config.transfer_group_index_by_layer == {"layer.0": 0, "layer.2": 1}
|
||||
first_blocks = [1, 2]
|
||||
third_blocks = [4]
|
||||
assert config.select_transfer_block_ids((first_blocks, [3], third_blocks)) == (
|
||||
first_blocks,
|
||||
third_blocks,
|
||||
)
|
||||
|
||||
|
||||
def new_sliding_window_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
|
||||
@@ -33,6 +33,9 @@ def _make_bare_scheduler(
|
||||
scheduler._block_size = 16
|
||||
scheduler._hash_block_size = hash_block_size
|
||||
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
|
||||
scheduler.kv_cache_config = SimpleNamespace(
|
||||
select_transfer_block_ids=lambda block_ids: tuple(block_ids)
|
||||
)
|
||||
scheduler.load_specs = {}
|
||||
scheduler._unfinished_request_ids = {"req-0"}
|
||||
scheduler._unfinished_requests = {}
|
||||
@@ -179,6 +182,20 @@ def _add_unfinished_request(
|
||||
)
|
||||
|
||||
|
||||
def test_update_state_excludes_nontransfer_groups():
|
||||
"""Store metadata must match the worker's registered cache groups."""
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler.kv_cache_config = SimpleNamespace(
|
||||
select_transfer_block_ids=lambda block_ids: (block_ids[0],)
|
||||
)
|
||||
request = SimpleNamespace(request_id="req-1")
|
||||
blocks = SimpleNamespace(get_block_ids=lambda: ([1, 2], [9]))
|
||||
|
||||
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=32)
|
||||
|
||||
assert scheduler._unfinished_requests["req-1"][1] == ([1, 2],)
|
||||
|
||||
|
||||
def _setup_decode_request(
|
||||
*,
|
||||
kv_role: str = "kv_consumer",
|
||||
|
||||
@@ -854,7 +854,10 @@ def test_post_process_zeroes_untransferred_tail():
|
||||
worker.device_kv_caches = {"attn.0": attn_cache, "mamba.0": mamba_cache}
|
||||
fa_group = MagicMock(layer_names=["attn.0"])
|
||||
ssm_group = MagicMock(layer_names=["mamba.0"])
|
||||
worker.kv_cache_config = MagicMock(kv_cache_groups=[fa_group, ssm_group])
|
||||
worker.kv_cache_config = MagicMock(
|
||||
kv_cache_groups=[fa_group, ssm_group],
|
||||
transfer_groups=[fa_group, ssm_group],
|
||||
)
|
||||
# The cached property filters mamba layers out of the permuted caches.
|
||||
attn_caches = NixlConnectorWorker._attention_kv_caches.func(worker)
|
||||
assert len(attn_caches) == 1 and attn_caches[0] is attn_cache
|
||||
@@ -1477,6 +1480,16 @@ def test_exchange_clipped_blocks_ssm_positional_states():
|
||||
assert clipped == ([1, 2, 3], [0, 5, 6, 7])
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_exchange_clipped_blocks_excludes_nontransfer_groups():
|
||||
"""Scheduler block tables must match the worker's registered regions."""
|
||||
sched = make_nixl_scheduler()
|
||||
sched.kv_cache_config = make_kv_cache_config(block_size=16, swa_enabled=True)
|
||||
sched.kv_cache_config.kv_cache_groups[1].enable_kv_transfer = False
|
||||
|
||||
assert sched.get_exchange_clipped_blocks(([1, 2], [9])) == ([1, 2],)
|
||||
|
||||
|
||||
# ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ─────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -524,6 +524,10 @@ def make_nixl_scheduler(
|
||||
sched = object.__new__(NixlConnectorScheduler)
|
||||
sched._has_mamba = has_mamba
|
||||
sched._is_hma_required = is_hma_required
|
||||
sched.kv_cache_config = make_kv_cache_config(
|
||||
block_size=16,
|
||||
mamba_enabled=has_mamba,
|
||||
)
|
||||
|
||||
if heartbeat:
|
||||
sched._heartbeat_by_engine = {}
|
||||
@@ -582,6 +586,10 @@ def make_nixl_push_scheduler(
|
||||
sched.side_channel_port = 5600
|
||||
sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled
|
||||
sched._has_mamba = has_mamba
|
||||
sched.kv_cache_config = make_kv_cache_config(
|
||||
block_size=16,
|
||||
mamba_enabled=has_mamba,
|
||||
)
|
||||
|
||||
# vllm_config is consulted for parallel_config.tensor_parallel_size.
|
||||
vllm_config = MagicMock()
|
||||
|
||||
@@ -97,7 +97,7 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
unsupported: list[str] = []
|
||||
cache_block_size = vllm_config.cache_config.block_size
|
||||
for g_idx, g in enumerate(kv_cache_config.kv_cache_groups):
|
||||
for g_idx, g in enumerate(kv_cache_config.transfer_groups):
|
||||
spec = g.kv_cache_spec
|
||||
if isinstance(spec, CrossAttentionSpec):
|
||||
unsupported.append(f"group {g_idx}: CrossAttentionSpec")
|
||||
@@ -110,7 +110,7 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
)
|
||||
pcp = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
dcp = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if len(kv_cache_config.kv_cache_groups) > 1 and pcp * dcp > 1:
|
||||
if len(kv_cache_config.transfer_groups) > 1 and pcp * dcp > 1:
|
||||
unsupported.append(
|
||||
f"PCP/DCP > 1 (pcp={pcp}, dcp={dcp}) with hybrid attention"
|
||||
)
|
||||
|
||||
@@ -69,6 +69,7 @@ class MooncakeStoreScheduler:
|
||||
kv_event_config and kv_event_config.enable_kv_cache_events
|
||||
)
|
||||
self.client = LookupKeyClient(vllm_config)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
|
||||
# Align with the engine's own scheduler_block_size and hash_block_size.
|
||||
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
|
||||
@@ -157,7 +158,9 @@ class MooncakeStoreScheduler:
|
||||
"""Update state after block allocation."""
|
||||
local_block_ids: tuple[list[int], ...] = ()
|
||||
if num_external_tokens > 0:
|
||||
local_block_ids = blocks.get_block_ids()
|
||||
local_block_ids = self.kv_cache_config.select_transfer_block_ids(
|
||||
blocks.get_block_ids()
|
||||
)
|
||||
|
||||
self._unfinished_requests[request.request_id] = (request, local_block_ids)
|
||||
self._unfinished_request_ids.add(request.request_id)
|
||||
@@ -220,12 +223,12 @@ class MooncakeStoreScheduler:
|
||||
request_tuple = self._unfinished_requests.get(request.req_id)
|
||||
request_real = request_tuple[0] # type: ignore[index]
|
||||
|
||||
if isinstance(request.block_ids, tuple):
|
||||
# Multi-group: preserve per-group structure.
|
||||
unfolded_block_ids = tuple(b.copy() for b in request.block_ids)
|
||||
else:
|
||||
# Single-group legacy: list[int] -> 1-tuple.
|
||||
unfolded_block_ids = (request.block_ids.copy(),)
|
||||
unfolded_block_ids = tuple(
|
||||
blocks.copy()
|
||||
for blocks in self.kv_cache_config.select_transfer_block_ids(
|
||||
request.block_ids
|
||||
)
|
||||
)
|
||||
|
||||
prefill_tokens = _new_req_prefill_tokens(request)
|
||||
request_tracker = RequestTracker(
|
||||
@@ -259,16 +262,17 @@ class MooncakeStoreScheduler:
|
||||
if can_process_cached:
|
||||
for i, req_id in enumerate(cached_reqs.req_ids):
|
||||
new_block_ids = cached_reqs.new_block_ids[i]
|
||||
if new_block_ids:
|
||||
new_block_ids = self.kv_cache_config.select_transfer_block_ids(
|
||||
new_block_ids
|
||||
)
|
||||
|
||||
req_meta = None
|
||||
if req_id in cached_reqs.resumed_req_ids:
|
||||
# Resumed after preemption
|
||||
if not new_block_ids:
|
||||
continue
|
||||
if isinstance(new_block_ids, tuple):
|
||||
new_block_ids = tuple(b.copy() for b in new_block_ids)
|
||||
else:
|
||||
new_block_ids = (new_block_ids.copy(),)
|
||||
new_block_ids = tuple(b.copy() for b in new_block_ids)
|
||||
load_spec = self.load_specs.pop(req_id, None)
|
||||
request_tuple = self._unfinished_requests.get(req_id)
|
||||
request_real = request_tuple[0] # type: ignore[index]
|
||||
|
||||
@@ -1497,7 +1497,7 @@ class MooncakeStoreWorker:
|
||||
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
|
||||
# self.block_size (= scheduler_block_size) so the coordinator's
|
||||
# ``block_size % hash_block_size == 0`` invariant holds.
|
||||
groups = list(kv_cache_config.kv_cache_groups)
|
||||
groups = list(kv_cache_config.transfer_groups)
|
||||
if len(groups) == 1 and groups[0].kv_cache_spec.block_size != self.block_size:
|
||||
g = groups[0]
|
||||
groups = [
|
||||
|
||||
@@ -90,12 +90,12 @@ class NixlBaseConnectorScheduler:
|
||||
# Also handle unlikely SW-only model case instead of checking num_groups>1.
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
self._has_mamba = any(
|
||||
isinstance(g.kv_cache_spec, MambaSpec)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
|
||||
logger.info("Initializing NIXL Scheduler %s", engine_id)
|
||||
@@ -131,7 +131,7 @@ class NixlBaseConnectorScheduler:
|
||||
(g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size)
|
||||
if isinstance(g.kv_cache_spec, SlidingWindowSpec)
|
||||
else (0, self.block_size)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
]
|
||||
# cdiv(n_tokens, block_size) gives blocks/window; add 1 to conservatively
|
||||
# account for boundary overlap eg window isn't fully aligned with blocks.
|
||||
@@ -146,7 +146,7 @@ class NixlBaseConnectorScheduler:
|
||||
g.kv_cache_spec.num_speculative_blocks
|
||||
if isinstance(g.kv_cache_spec, MambaSpec)
|
||||
else None
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
]
|
||||
# Only "all" mode keeps a state per block position; the other modes
|
||||
# keep a single running state in the last non-speculative slot.
|
||||
@@ -260,8 +260,11 @@ class NixlBaseConnectorScheduler:
|
||||
for per-step partial lists (host-buffer save), where the SSM strip
|
||||
does not apply.
|
||||
"""
|
||||
if len(block_ids) == 0 or not self._is_hma_required:
|
||||
# No blocks to clip eg Full prefix cache hit or not a hybrid model.
|
||||
if len(block_ids) == 0:
|
||||
# No blocks to clip, e.g. a full prefix cache hit.
|
||||
return block_ids
|
||||
block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids)
|
||||
if not self._is_hma_required:
|
||||
return block_ids
|
||||
# NOTE (NickLucche) This logic is currently handled at the connector level
|
||||
# because offloading connectors might want to receive the whole sequence even
|
||||
|
||||
@@ -300,13 +300,13 @@ class NixlBaseConnectorWorker:
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self._layer_specs = {
|
||||
layer: group.kv_cache_spec
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
for group in kv_cache_config.transfer_groups
|
||||
for layer in group.layer_names
|
||||
}
|
||||
|
||||
@@ -318,7 +318,7 @@ class NixlBaseConnectorWorker:
|
||||
self._conv_decomp: MambaConvSplitInfo | None = None
|
||||
self._has_mamba = any(
|
||||
isinstance(g.kv_cache_spec, MambaSpec)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
if self._has_mamba:
|
||||
assert self._is_hma_required
|
||||
@@ -554,7 +554,7 @@ class NixlBaseConnectorWorker:
|
||||
# Unwrap UniformTypeKVCacheSpecs to get the representative spec type
|
||||
self._group_spec_types = tuple(
|
||||
get_representative_spec_type(g.kv_cache_spec)
|
||||
for g in self.kv_cache_config.kv_cache_groups
|
||||
for g in self.kv_cache_config.transfer_groups
|
||||
)
|
||||
|
||||
# Per-region MLA flag, 1:1 with block_len_per_layer. True -> REPLICATE
|
||||
@@ -1906,7 +1906,7 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
mamba_layers = {
|
||||
name
|
||||
for g, group in enumerate(self.kv_cache_config.kv_cache_groups)
|
||||
for g, group in enumerate(self.kv_cache_config.transfer_groups)
|
||||
if _is_ssm_spec(self._group_spec_types[g])
|
||||
for name in group.layer_names
|
||||
}
|
||||
@@ -2361,7 +2361,7 @@ class NixlBaseConnectorWorker:
|
||||
return block_ids
|
||||
block_arange = np.arange(0, ratio).reshape(1, -1)
|
||||
# Mamba blocks have no logical<>physical discrepancy (block-size=1)
|
||||
group_specs = self.kv_cache_config.kv_cache_groups
|
||||
group_specs = self.kv_cache_config.transfer_groups
|
||||
physical_block_ids = []
|
||||
for i, group in enumerate(block_ids):
|
||||
spec = group_specs[i].kv_cache_spec
|
||||
|
||||
@@ -290,7 +290,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
assert (
|
||||
len(remote_block_ids)
|
||||
== len(local_block_ids)
|
||||
== len(self.kv_cache_config.kv_cache_groups)
|
||||
== len(self.kv_cache_config.transfer_groups)
|
||||
)
|
||||
local_block_ids, remote_block_ids = self._apply_prefix_caching(
|
||||
decode_block_ids=local_block_ids,
|
||||
|
||||
@@ -5,10 +5,11 @@ from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from collections import Counter
|
||||
from collections.abc import Collection
|
||||
from collections.abc import Collection, Sequence
|
||||
from dataclasses import dataclass, fields, replace
|
||||
from enum import Enum, IntEnum
|
||||
from fractions import Fraction
|
||||
from functools import cached_property
|
||||
from math import prod
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
@@ -1143,6 +1144,8 @@ class KVCacheGroupSpec:
|
||||
kv_cache_spec: KVCacheSpec
|
||||
# Whether this group contains EAGLE/MTP draft attention layers.
|
||||
is_eagle_group: bool = False
|
||||
# Whether this group is part of the externally transferable KV state.
|
||||
enable_kv_transfer: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1168,6 +1171,42 @@ class KVCacheConfig:
|
||||
kv_cache_layout: str | None = None
|
||||
"""The KV cache layout resolved by the engine core, adopted by all workers."""
|
||||
|
||||
@cached_property
|
||||
def transfer_group_ids(self) -> tuple[int, ...]:
|
||||
"""IDs of cache groups that participate in external KV transfer."""
|
||||
return tuple(
|
||||
group_id
|
||||
for group_id, group in enumerate(self.kv_cache_groups)
|
||||
if group.enable_kv_transfer
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def transfer_groups(self) -> tuple[KVCacheGroupSpec, ...]:
|
||||
"""Cache groups that participate in external KV transfer."""
|
||||
return tuple(
|
||||
self.kv_cache_groups[group_id] for group_id in self.transfer_group_ids
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def transfer_group_index_by_layer(self) -> dict[str, int]:
|
||||
"""Transfer-group tuple index for each participating layer."""
|
||||
return {
|
||||
layer_name: group_index
|
||||
for group_index, group in enumerate(self.transfer_groups)
|
||||
for layer_name in group.layer_names
|
||||
}
|
||||
|
||||
def select_transfer_block_ids(
|
||||
self, block_ids: Sequence[list[int]]
|
||||
) -> tuple[list[int], ...]:
|
||||
"""Select block IDs for externally transferable cache groups."""
|
||||
if len(block_ids) != len(self.kv_cache_groups):
|
||||
raise ValueError(
|
||||
f"Expected {len(self.kv_cache_groups)} KV cache groups, "
|
||||
f"got {len(block_ids)}."
|
||||
)
|
||||
return tuple(block_ids[group_id] for group_id in self.transfer_group_ids)
|
||||
|
||||
@property
|
||||
def has_mamba_layers(self) -> bool:
|
||||
return any(isinstance(g.kv_cache_spec, MambaSpec) for g in self.kv_cache_groups)
|
||||
|
||||
Reference in New Issue
Block a user