[1/N][KV Connector] Identify externally transferable KV cache groups (#53779)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
@@ -140,6 +140,30 @@ def new_kv_cache_spec(
|
||||
)
|
||||
|
||||
|
||||
def test_kv_cache_config_selects_only_transferable_groups():
|
||||
"""Connectors must see a stable projection of transfer-eligible groups."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["layer.0"], new_kv_cache_spec()),
|
||||
KVCacheGroupSpec(["layer.1"], new_kv_cache_spec(), enable_kv_transfer=False),
|
||||
KVCacheGroupSpec(["layer.2"], new_kv_cache_spec()),
|
||||
]
|
||||
config = KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
|
||||
assert config.transfer_group_ids == (0, 2)
|
||||
assert config.transfer_groups == (groups[0], groups[2])
|
||||
assert config.transfer_group_index_by_layer == {"layer.0": 0, "layer.2": 1}
|
||||
first_blocks = [1, 2]
|
||||
third_blocks = [4]
|
||||
assert config.select_transfer_block_ids((first_blocks, [3], third_blocks)) == (
|
||||
first_blocks,
|
||||
third_blocks,
|
||||
)
|
||||
|
||||
|
||||
def new_sliding_window_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
|
||||
Reference in New Issue
Block a user