[1/N][KV Connector] Identify externally transferable KV cache groups (#53779)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Matthew Bonanni
2026-08-27 11:21:24 -04:00
committed by GitHub
parent c7b0467c25
commit d85708f7a4
11 changed files with 137 additions and 29 deletions
+40 -1
View File
@@ -5,10 +5,11 @@ from __future__ import annotations
import copy
from collections import Counter
from collections.abc import Collection
from collections.abc import Collection, Sequence
from dataclasses import dataclass, fields, replace
from enum import Enum, IntEnum
from fractions import Fraction
from functools import cached_property
from math import prod
from typing import TYPE_CHECKING, TypeVar
@@ -1143,6 +1144,8 @@ class KVCacheGroupSpec:
kv_cache_spec: KVCacheSpec
# Whether this group contains EAGLE/MTP draft attention layers.
is_eagle_group: bool = False
# Whether this group is part of the externally transferable KV state.
enable_kv_transfer: bool = True
@dataclass
@@ -1168,6 +1171,42 @@ class KVCacheConfig:
kv_cache_layout: str | None = None
"""The KV cache layout resolved by the engine core, adopted by all workers."""
@cached_property
def transfer_group_ids(self) -> tuple[int, ...]:
"""IDs of cache groups that participate in external KV transfer."""
return tuple(
group_id
for group_id, group in enumerate(self.kv_cache_groups)
if group.enable_kv_transfer
)
@cached_property
def transfer_groups(self) -> tuple[KVCacheGroupSpec, ...]:
"""Cache groups that participate in external KV transfer."""
return tuple(
self.kv_cache_groups[group_id] for group_id in self.transfer_group_ids
)
@cached_property
def transfer_group_index_by_layer(self) -> dict[str, int]:
"""Transfer-group tuple index for each participating layer."""
return {
layer_name: group_index
for group_index, group in enumerate(self.transfer_groups)
for layer_name in group.layer_names
}
def select_transfer_block_ids(
self, block_ids: Sequence[list[int]]
) -> tuple[list[int], ...]:
"""Select block IDs for externally transferable cache groups."""
if len(block_ids) != len(self.kv_cache_groups):
raise ValueError(
f"Expected {len(self.kv_cache_groups)} KV cache groups, "
f"got {len(block_ids)}."
)
return tuple(block_ids[group_id] for group_id in self.transfer_group_ids)
@property
def has_mamba_layers(self) -> bool:
return any(isinstance(g.kv_cache_spec, MambaSpec) for g in self.kv_cache_groups)