[1/N][KV Connector] Identify externally transferable KV cache groups (#53779)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Matthew Bonanni
2026-08-27 11:21:24 -04:00
committed by GitHub
parent c7b0467c25
commit d85708f7a4
11 changed files with 137 additions and 29 deletions
@@ -97,7 +97,7 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
unsupported: list[str] = []
cache_block_size = vllm_config.cache_config.block_size
for g_idx, g in enumerate(kv_cache_config.kv_cache_groups):
for g_idx, g in enumerate(kv_cache_config.transfer_groups):
spec = g.kv_cache_spec
if isinstance(spec, CrossAttentionSpec):
unsupported.append(f"group {g_idx}: CrossAttentionSpec")
@@ -110,7 +110,7 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
)
pcp = vllm_config.parallel_config.prefill_context_parallel_size
dcp = vllm_config.parallel_config.decode_context_parallel_size
if len(kv_cache_config.kv_cache_groups) > 1 and pcp * dcp > 1:
if len(kv_cache_config.transfer_groups) > 1 and pcp * dcp > 1:
unsupported.append(
f"PCP/DCP > 1 (pcp={pcp}, dcp={dcp}) with hybrid attention"
)
@@ -69,6 +69,7 @@ class MooncakeStoreScheduler:
kv_event_config and kv_event_config.enable_kv_cache_events
)
self.client = LookupKeyClient(vllm_config)
self.kv_cache_config = kv_cache_config
# Align with the engine's own scheduler_block_size and hash_block_size.
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
@@ -157,7 +158,9 @@ class MooncakeStoreScheduler:
"""Update state after block allocation."""
local_block_ids: tuple[list[int], ...] = ()
if num_external_tokens > 0:
local_block_ids = blocks.get_block_ids()
local_block_ids = self.kv_cache_config.select_transfer_block_ids(
blocks.get_block_ids()
)
self._unfinished_requests[request.request_id] = (request, local_block_ids)
self._unfinished_request_ids.add(request.request_id)
@@ -220,12 +223,12 @@ class MooncakeStoreScheduler:
request_tuple = self._unfinished_requests.get(request.req_id)
request_real = request_tuple[0] # type: ignore[index]
if isinstance(request.block_ids, tuple):
# Multi-group: preserve per-group structure.
unfolded_block_ids = tuple(b.copy() for b in request.block_ids)
else:
# Single-group legacy: list[int] -> 1-tuple.
unfolded_block_ids = (request.block_ids.copy(),)
unfolded_block_ids = tuple(
blocks.copy()
for blocks in self.kv_cache_config.select_transfer_block_ids(
request.block_ids
)
)
prefill_tokens = _new_req_prefill_tokens(request)
request_tracker = RequestTracker(
@@ -259,16 +262,17 @@ class MooncakeStoreScheduler:
if can_process_cached:
for i, req_id in enumerate(cached_reqs.req_ids):
new_block_ids = cached_reqs.new_block_ids[i]
if new_block_ids:
new_block_ids = self.kv_cache_config.select_transfer_block_ids(
new_block_ids
)
req_meta = None
if req_id in cached_reqs.resumed_req_ids:
# Resumed after preemption
if not new_block_ids:
continue
if isinstance(new_block_ids, tuple):
new_block_ids = tuple(b.copy() for b in new_block_ids)
else:
new_block_ids = (new_block_ids.copy(),)
new_block_ids = tuple(b.copy() for b in new_block_ids)
load_spec = self.load_specs.pop(req_id, None)
request_tuple = self._unfinished_requests.get(req_id)
request_real = request_tuple[0] # type: ignore[index]
@@ -1497,7 +1497,7 @@ class MooncakeStoreWorker:
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
# self.block_size (= scheduler_block_size) so the coordinator's
# ``block_size % hash_block_size == 0`` invariant holds.
groups = list(kv_cache_config.kv_cache_groups)
groups = list(kv_cache_config.transfer_groups)
if len(groups) == 1 and groups[0].kv_cache_spec.block_size != self.block_size:
g = groups[0]
groups = [
@@ -90,12 +90,12 @@ class NixlBaseConnectorScheduler:
# Also handle unlikely SW-only model case instead of checking num_groups>1.
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
)
)
self._has_mamba = any(
isinstance(g.kv_cache_spec, MambaSpec)
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
)
logger.info("Initializing NIXL Scheduler %s", engine_id)
@@ -131,7 +131,7 @@ class NixlBaseConnectorScheduler:
(g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size)
if isinstance(g.kv_cache_spec, SlidingWindowSpec)
else (0, self.block_size)
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
]
# cdiv(n_tokens, block_size) gives blocks/window; add 1 to conservatively
# account for boundary overlap eg window isn't fully aligned with blocks.
@@ -146,7 +146,7 @@ class NixlBaseConnectorScheduler:
g.kv_cache_spec.num_speculative_blocks
if isinstance(g.kv_cache_spec, MambaSpec)
else None
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
]
# Only "all" mode keeps a state per block position; the other modes
# keep a single running state in the last non-speculative slot.
@@ -260,8 +260,11 @@ class NixlBaseConnectorScheduler:
for per-step partial lists (host-buffer save), where the SSM strip
does not apply.
"""
if len(block_ids) == 0 or not self._is_hma_required:
# No blocks to clip eg Full prefix cache hit or not a hybrid model.
if len(block_ids) == 0:
# No blocks to clip, e.g. a full prefix cache hit.
return block_ids
block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids)
if not self._is_hma_required:
return block_ids
# NOTE (NickLucche) This logic is currently handled at the connector level
# because offloading connectors might want to receive the whole sequence even
@@ -300,13 +300,13 @@ class NixlBaseConnectorWorker:
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
)
)
self.kv_cache_config = kv_cache_config
self._layer_specs = {
layer: group.kv_cache_spec
for group in kv_cache_config.kv_cache_groups
for group in kv_cache_config.transfer_groups
for layer in group.layer_names
}
@@ -318,7 +318,7 @@ class NixlBaseConnectorWorker:
self._conv_decomp: MambaConvSplitInfo | None = None
self._has_mamba = any(
isinstance(g.kv_cache_spec, MambaSpec)
for g in kv_cache_config.kv_cache_groups
for g in kv_cache_config.transfer_groups
)
if self._has_mamba:
assert self._is_hma_required
@@ -554,7 +554,7 @@ class NixlBaseConnectorWorker:
# Unwrap UniformTypeKVCacheSpecs to get the representative spec type
self._group_spec_types = tuple(
get_representative_spec_type(g.kv_cache_spec)
for g in self.kv_cache_config.kv_cache_groups
for g in self.kv_cache_config.transfer_groups
)
# Per-region MLA flag, 1:1 with block_len_per_layer. True -> REPLICATE
@@ -1906,7 +1906,7 @@ class NixlBaseConnectorWorker:
)
mamba_layers = {
name
for g, group in enumerate(self.kv_cache_config.kv_cache_groups)
for g, group in enumerate(self.kv_cache_config.transfer_groups)
if _is_ssm_spec(self._group_spec_types[g])
for name in group.layer_names
}
@@ -2361,7 +2361,7 @@ class NixlBaseConnectorWorker:
return block_ids
block_arange = np.arange(0, ratio).reshape(1, -1)
# Mamba blocks have no logical<>physical discrepancy (block-size=1)
group_specs = self.kv_cache_config.kv_cache_groups
group_specs = self.kv_cache_config.transfer_groups
physical_block_ids = []
for i, group in enumerate(block_ids):
spec = group_specs[i].kv_cache_spec
@@ -290,7 +290,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
assert (
len(remote_block_ids)
== len(local_block_ids)
== len(self.kv_cache_config.kv_cache_groups)
== len(self.kv_cache_config.transfer_groups)
)
local_block_ids, remote_block_ids = self._apply_prefix_caching(
decode_block_ids=local_block_ids,