[1/N][KV Connector] Identify externally transferable KV cache groups (#53779)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
@@ -97,7 +97,7 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
unsupported: list[str] = []
|
||||
cache_block_size = vllm_config.cache_config.block_size
|
||||
for g_idx, g in enumerate(kv_cache_config.kv_cache_groups):
|
||||
for g_idx, g in enumerate(kv_cache_config.transfer_groups):
|
||||
spec = g.kv_cache_spec
|
||||
if isinstance(spec, CrossAttentionSpec):
|
||||
unsupported.append(f"group {g_idx}: CrossAttentionSpec")
|
||||
@@ -110,7 +110,7 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
)
|
||||
pcp = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
dcp = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if len(kv_cache_config.kv_cache_groups) > 1 and pcp * dcp > 1:
|
||||
if len(kv_cache_config.transfer_groups) > 1 and pcp * dcp > 1:
|
||||
unsupported.append(
|
||||
f"PCP/DCP > 1 (pcp={pcp}, dcp={dcp}) with hybrid attention"
|
||||
)
|
||||
|
||||
@@ -69,6 +69,7 @@ class MooncakeStoreScheduler:
|
||||
kv_event_config and kv_event_config.enable_kv_cache_events
|
||||
)
|
||||
self.client = LookupKeyClient(vllm_config)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
|
||||
# Align with the engine's own scheduler_block_size and hash_block_size.
|
||||
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
|
||||
@@ -157,7 +158,9 @@ class MooncakeStoreScheduler:
|
||||
"""Update state after block allocation."""
|
||||
local_block_ids: tuple[list[int], ...] = ()
|
||||
if num_external_tokens > 0:
|
||||
local_block_ids = blocks.get_block_ids()
|
||||
local_block_ids = self.kv_cache_config.select_transfer_block_ids(
|
||||
blocks.get_block_ids()
|
||||
)
|
||||
|
||||
self._unfinished_requests[request.request_id] = (request, local_block_ids)
|
||||
self._unfinished_request_ids.add(request.request_id)
|
||||
@@ -220,12 +223,12 @@ class MooncakeStoreScheduler:
|
||||
request_tuple = self._unfinished_requests.get(request.req_id)
|
||||
request_real = request_tuple[0] # type: ignore[index]
|
||||
|
||||
if isinstance(request.block_ids, tuple):
|
||||
# Multi-group: preserve per-group structure.
|
||||
unfolded_block_ids = tuple(b.copy() for b in request.block_ids)
|
||||
else:
|
||||
# Single-group legacy: list[int] -> 1-tuple.
|
||||
unfolded_block_ids = (request.block_ids.copy(),)
|
||||
unfolded_block_ids = tuple(
|
||||
blocks.copy()
|
||||
for blocks in self.kv_cache_config.select_transfer_block_ids(
|
||||
request.block_ids
|
||||
)
|
||||
)
|
||||
|
||||
prefill_tokens = _new_req_prefill_tokens(request)
|
||||
request_tracker = RequestTracker(
|
||||
@@ -259,16 +262,17 @@ class MooncakeStoreScheduler:
|
||||
if can_process_cached:
|
||||
for i, req_id in enumerate(cached_reqs.req_ids):
|
||||
new_block_ids = cached_reqs.new_block_ids[i]
|
||||
if new_block_ids:
|
||||
new_block_ids = self.kv_cache_config.select_transfer_block_ids(
|
||||
new_block_ids
|
||||
)
|
||||
|
||||
req_meta = None
|
||||
if req_id in cached_reqs.resumed_req_ids:
|
||||
# Resumed after preemption
|
||||
if not new_block_ids:
|
||||
continue
|
||||
if isinstance(new_block_ids, tuple):
|
||||
new_block_ids = tuple(b.copy() for b in new_block_ids)
|
||||
else:
|
||||
new_block_ids = (new_block_ids.copy(),)
|
||||
new_block_ids = tuple(b.copy() for b in new_block_ids)
|
||||
load_spec = self.load_specs.pop(req_id, None)
|
||||
request_tuple = self._unfinished_requests.get(req_id)
|
||||
request_real = request_tuple[0] # type: ignore[index]
|
||||
|
||||
@@ -1497,7 +1497,7 @@ class MooncakeStoreWorker:
|
||||
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
|
||||
# self.block_size (= scheduler_block_size) so the coordinator's
|
||||
# ``block_size % hash_block_size == 0`` invariant holds.
|
||||
groups = list(kv_cache_config.kv_cache_groups)
|
||||
groups = list(kv_cache_config.transfer_groups)
|
||||
if len(groups) == 1 and groups[0].kv_cache_spec.block_size != self.block_size:
|
||||
g = groups[0]
|
||||
groups = [
|
||||
|
||||
@@ -90,12 +90,12 @@ class NixlBaseConnectorScheduler:
|
||||
# Also handle unlikely SW-only model case instead of checking num_groups>1.
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
self._has_mamba = any(
|
||||
isinstance(g.kv_cache_spec, MambaSpec)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
|
||||
logger.info("Initializing NIXL Scheduler %s", engine_id)
|
||||
@@ -131,7 +131,7 @@ class NixlBaseConnectorScheduler:
|
||||
(g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size)
|
||||
if isinstance(g.kv_cache_spec, SlidingWindowSpec)
|
||||
else (0, self.block_size)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
]
|
||||
# cdiv(n_tokens, block_size) gives blocks/window; add 1 to conservatively
|
||||
# account for boundary overlap eg window isn't fully aligned with blocks.
|
||||
@@ -146,7 +146,7 @@ class NixlBaseConnectorScheduler:
|
||||
g.kv_cache_spec.num_speculative_blocks
|
||||
if isinstance(g.kv_cache_spec, MambaSpec)
|
||||
else None
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
]
|
||||
# Only "all" mode keeps a state per block position; the other modes
|
||||
# keep a single running state in the last non-speculative slot.
|
||||
@@ -260,8 +260,11 @@ class NixlBaseConnectorScheduler:
|
||||
for per-step partial lists (host-buffer save), where the SSM strip
|
||||
does not apply.
|
||||
"""
|
||||
if len(block_ids) == 0 or not self._is_hma_required:
|
||||
# No blocks to clip eg Full prefix cache hit or not a hybrid model.
|
||||
if len(block_ids) == 0:
|
||||
# No blocks to clip, e.g. a full prefix cache hit.
|
||||
return block_ids
|
||||
block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids)
|
||||
if not self._is_hma_required:
|
||||
return block_ids
|
||||
# NOTE (NickLucche) This logic is currently handled at the connector level
|
||||
# because offloading connectors might want to receive the whole sequence even
|
||||
|
||||
@@ -300,13 +300,13 @@ class NixlBaseConnectorWorker:
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self._layer_specs = {
|
||||
layer: group.kv_cache_spec
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
for group in kv_cache_config.transfer_groups
|
||||
for layer in group.layer_names
|
||||
}
|
||||
|
||||
@@ -318,7 +318,7 @@ class NixlBaseConnectorWorker:
|
||||
self._conv_decomp: MambaConvSplitInfo | None = None
|
||||
self._has_mamba = any(
|
||||
isinstance(g.kv_cache_spec, MambaSpec)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
if self._has_mamba:
|
||||
assert self._is_hma_required
|
||||
@@ -554,7 +554,7 @@ class NixlBaseConnectorWorker:
|
||||
# Unwrap UniformTypeKVCacheSpecs to get the representative spec type
|
||||
self._group_spec_types = tuple(
|
||||
get_representative_spec_type(g.kv_cache_spec)
|
||||
for g in self.kv_cache_config.kv_cache_groups
|
||||
for g in self.kv_cache_config.transfer_groups
|
||||
)
|
||||
|
||||
# Per-region MLA flag, 1:1 with block_len_per_layer. True -> REPLICATE
|
||||
@@ -1906,7 +1906,7 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
mamba_layers = {
|
||||
name
|
||||
for g, group in enumerate(self.kv_cache_config.kv_cache_groups)
|
||||
for g, group in enumerate(self.kv_cache_config.transfer_groups)
|
||||
if _is_ssm_spec(self._group_spec_types[g])
|
||||
for name in group.layer_names
|
||||
}
|
||||
@@ -2361,7 +2361,7 @@ class NixlBaseConnectorWorker:
|
||||
return block_ids
|
||||
block_arange = np.arange(0, ratio).reshape(1, -1)
|
||||
# Mamba blocks have no logical<>physical discrepancy (block-size=1)
|
||||
group_specs = self.kv_cache_config.kv_cache_groups
|
||||
group_specs = self.kv_cache_config.transfer_groups
|
||||
physical_block_ids = []
|
||||
for i, group in enumerate(block_ids):
|
||||
spec = group_specs[i].kv_cache_spec
|
||||
|
||||
@@ -290,7 +290,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
assert (
|
||||
len(remote_block_ids)
|
||||
== len(local_block_ids)
|
||||
== len(self.kv_cache_config.kv_cache_groups)
|
||||
== len(self.kv_cache_config.transfer_groups)
|
||||
)
|
||||
local_block_ids, remote_block_ids = self._apply_prefix_caching(
|
||||
decode_block_ids=local_block_ids,
|
||||
|
||||
Reference in New Issue
Block a user