[1/N][KV Connector] Identify externally transferable KV cache groups (#53779)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
@@ -5,10 +5,11 @@ from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from collections import Counter
|
||||
from collections.abc import Collection
|
||||
from collections.abc import Collection, Sequence
|
||||
from dataclasses import dataclass, fields, replace
|
||||
from enum import Enum, IntEnum
|
||||
from fractions import Fraction
|
||||
from functools import cached_property
|
||||
from math import prod
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
@@ -1143,6 +1144,8 @@ class KVCacheGroupSpec:
|
||||
kv_cache_spec: KVCacheSpec
|
||||
# Whether this group contains EAGLE/MTP draft attention layers.
|
||||
is_eagle_group: bool = False
|
||||
# Whether this group is part of the externally transferable KV state.
|
||||
enable_kv_transfer: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1168,6 +1171,42 @@ class KVCacheConfig:
|
||||
kv_cache_layout: str | None = None
|
||||
"""The KV cache layout resolved by the engine core, adopted by all workers."""
|
||||
|
||||
@cached_property
|
||||
def transfer_group_ids(self) -> tuple[int, ...]:
|
||||
"""IDs of cache groups that participate in external KV transfer."""
|
||||
return tuple(
|
||||
group_id
|
||||
for group_id, group in enumerate(self.kv_cache_groups)
|
||||
if group.enable_kv_transfer
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def transfer_groups(self) -> tuple[KVCacheGroupSpec, ...]:
|
||||
"""Cache groups that participate in external KV transfer."""
|
||||
return tuple(
|
||||
self.kv_cache_groups[group_id] for group_id in self.transfer_group_ids
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def transfer_group_index_by_layer(self) -> dict[str, int]:
|
||||
"""Transfer-group tuple index for each participating layer."""
|
||||
return {
|
||||
layer_name: group_index
|
||||
for group_index, group in enumerate(self.transfer_groups)
|
||||
for layer_name in group.layer_names
|
||||
}
|
||||
|
||||
def select_transfer_block_ids(
|
||||
self, block_ids: Sequence[list[int]]
|
||||
) -> tuple[list[int], ...]:
|
||||
"""Select block IDs for externally transferable cache groups."""
|
||||
if len(block_ids) != len(self.kv_cache_groups):
|
||||
raise ValueError(
|
||||
f"Expected {len(self.kv_cache_groups)} KV cache groups, "
|
||||
f"got {len(block_ids)}."
|
||||
)
|
||||
return tuple(block_ids[group_id] for group_id in self.transfer_group_ids)
|
||||
|
||||
@property
|
||||
def has_mamba_layers(self) -> bool:
|
||||
return any(isinstance(g.kv_cache_spec, MambaSpec) for g in self.kv_cache_groups)
|
||||
|
||||
Reference in New Issue
Block a user