[1/N][KV Connector] Identify externally transferable KV cache groups (#53779)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Matthew Bonanni
2026-08-27 11:21:24 -04:00
committed by GitHub
parent c7b0467c25
commit d85708f7a4
11 changed files with 137 additions and 29 deletions
+24
View File
@@ -140,6 +140,30 @@ def new_kv_cache_spec(
)
def test_kv_cache_config_selects_only_transferable_groups():
"""Connectors must see a stable projection of transfer-eligible groups."""
groups = [
KVCacheGroupSpec(["layer.0"], new_kv_cache_spec()),
KVCacheGroupSpec(["layer.1"], new_kv_cache_spec(), enable_kv_transfer=False),
KVCacheGroupSpec(["layer.2"], new_kv_cache_spec()),
]
config = KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=groups,
)
assert config.transfer_group_ids == (0, 2)
assert config.transfer_groups == (groups[0], groups[2])
assert config.transfer_group_index_by_layer == {"layer.0": 0, "layer.2": 1}
first_blocks = [1, 2]
third_blocks = [4]
assert config.select_transfer_block_ids((first_blocks, [3], third_blocks)) == (
first_blocks,
third_blocks,
)
def new_sliding_window_spec(
block_size=16,
num_kv_heads=2,
@@ -33,6 +33,9 @@ def _make_bare_scheduler(
scheduler._block_size = 16
scheduler._hash_block_size = hash_block_size
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
scheduler.kv_cache_config = SimpleNamespace(
select_transfer_block_ids=lambda block_ids: tuple(block_ids)
)
scheduler.load_specs = {}
scheduler._unfinished_request_ids = {"req-0"}
scheduler._unfinished_requests = {}
@@ -179,6 +182,20 @@ def _add_unfinished_request(
)
def test_update_state_excludes_nontransfer_groups():
"""Store metadata must match the worker's registered cache groups."""
scheduler = _make_bare_scheduler()
scheduler.kv_cache_config = SimpleNamespace(
select_transfer_block_ids=lambda block_ids: (block_ids[0],)
)
request = SimpleNamespace(request_id="req-1")
blocks = SimpleNamespace(get_block_ids=lambda: ([1, 2], [9]))
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=32)
assert scheduler._unfinished_requests["req-1"][1] == ([1, 2],)
def _setup_decode_request(
*,
kv_role: str = "kv_consumer",
@@ -854,7 +854,10 @@ def test_post_process_zeroes_untransferred_tail():
worker.device_kv_caches = {"attn.0": attn_cache, "mamba.0": mamba_cache}
fa_group = MagicMock(layer_names=["attn.0"])
ssm_group = MagicMock(layer_names=["mamba.0"])
worker.kv_cache_config = MagicMock(kv_cache_groups=[fa_group, ssm_group])
worker.kv_cache_config = MagicMock(
kv_cache_groups=[fa_group, ssm_group],
transfer_groups=[fa_group, ssm_group],
)
# The cached property filters mamba layers out of the permuted caches.
attn_caches = NixlConnectorWorker._attention_kv_caches.func(worker)
assert len(attn_caches) == 1 and attn_caches[0] is attn_cache
@@ -1477,6 +1480,16 @@ def test_exchange_clipped_blocks_ssm_positional_states():
assert clipped == ([1, 2, 3], [0, 5, 6, 7])
@pytest.mark.cpu_test
def test_exchange_clipped_blocks_excludes_nontransfer_groups():
"""Scheduler block tables must match the worker's registered regions."""
sched = make_nixl_scheduler()
sched.kv_cache_config = make_kv_cache_config(block_size=16, swa_enabled=True)
sched.kv_cache_config.kv_cache_groups[1].enable_kv_transfer = False
assert sched.get_exchange_clipped_blocks(([1, 2], [9])) == ([1, 2],)
# ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ─────────────────────
+8
View File
@@ -524,6 +524,10 @@ def make_nixl_scheduler(
sched = object.__new__(NixlConnectorScheduler)
sched._has_mamba = has_mamba
sched._is_hma_required = is_hma_required
sched.kv_cache_config = make_kv_cache_config(
block_size=16,
mamba_enabled=has_mamba,
)
if heartbeat:
sched._heartbeat_by_engine = {}
@@ -582,6 +586,10 @@ def make_nixl_push_scheduler(
sched.side_channel_port = 5600
sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled
sched._has_mamba = has_mamba
sched.kv_cache_config = make_kv_cache_config(
block_size=16,
mamba_enabled=has_mamba,
)
# vllm_config is consulted for parallel_config.tensor_parallel_size.
vllm_config = MagicMock()