[1/N][KV Connector] Identify externally transferable KV cache groups (#53779)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
@@ -140,6 +140,30 @@ def new_kv_cache_spec(
|
||||
)
|
||||
|
||||
|
||||
def test_kv_cache_config_selects_only_transferable_groups():
|
||||
"""Connectors must see a stable projection of transfer-eligible groups."""
|
||||
groups = [
|
||||
KVCacheGroupSpec(["layer.0"], new_kv_cache_spec()),
|
||||
KVCacheGroupSpec(["layer.1"], new_kv_cache_spec(), enable_kv_transfer=False),
|
||||
KVCacheGroupSpec(["layer.2"], new_kv_cache_spec()),
|
||||
]
|
||||
config = KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
|
||||
assert config.transfer_group_ids == (0, 2)
|
||||
assert config.transfer_groups == (groups[0], groups[2])
|
||||
assert config.transfer_group_index_by_layer == {"layer.0": 0, "layer.2": 1}
|
||||
first_blocks = [1, 2]
|
||||
third_blocks = [4]
|
||||
assert config.select_transfer_block_ids((first_blocks, [3], third_blocks)) == (
|
||||
first_blocks,
|
||||
third_blocks,
|
||||
)
|
||||
|
||||
|
||||
def new_sliding_window_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
|
||||
@@ -33,6 +33,9 @@ def _make_bare_scheduler(
|
||||
scheduler._block_size = 16
|
||||
scheduler._hash_block_size = hash_block_size
|
||||
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
|
||||
scheduler.kv_cache_config = SimpleNamespace(
|
||||
select_transfer_block_ids=lambda block_ids: tuple(block_ids)
|
||||
)
|
||||
scheduler.load_specs = {}
|
||||
scheduler._unfinished_request_ids = {"req-0"}
|
||||
scheduler._unfinished_requests = {}
|
||||
@@ -179,6 +182,20 @@ def _add_unfinished_request(
|
||||
)
|
||||
|
||||
|
||||
def test_update_state_excludes_nontransfer_groups():
|
||||
"""Store metadata must match the worker's registered cache groups."""
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler.kv_cache_config = SimpleNamespace(
|
||||
select_transfer_block_ids=lambda block_ids: (block_ids[0],)
|
||||
)
|
||||
request = SimpleNamespace(request_id="req-1")
|
||||
blocks = SimpleNamespace(get_block_ids=lambda: ([1, 2], [9]))
|
||||
|
||||
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=32)
|
||||
|
||||
assert scheduler._unfinished_requests["req-1"][1] == ([1, 2],)
|
||||
|
||||
|
||||
def _setup_decode_request(
|
||||
*,
|
||||
kv_role: str = "kv_consumer",
|
||||
|
||||
@@ -854,7 +854,10 @@ def test_post_process_zeroes_untransferred_tail():
|
||||
worker.device_kv_caches = {"attn.0": attn_cache, "mamba.0": mamba_cache}
|
||||
fa_group = MagicMock(layer_names=["attn.0"])
|
||||
ssm_group = MagicMock(layer_names=["mamba.0"])
|
||||
worker.kv_cache_config = MagicMock(kv_cache_groups=[fa_group, ssm_group])
|
||||
worker.kv_cache_config = MagicMock(
|
||||
kv_cache_groups=[fa_group, ssm_group],
|
||||
transfer_groups=[fa_group, ssm_group],
|
||||
)
|
||||
# The cached property filters mamba layers out of the permuted caches.
|
||||
attn_caches = NixlConnectorWorker._attention_kv_caches.func(worker)
|
||||
assert len(attn_caches) == 1 and attn_caches[0] is attn_cache
|
||||
@@ -1477,6 +1480,16 @@ def test_exchange_clipped_blocks_ssm_positional_states():
|
||||
assert clipped == ([1, 2, 3], [0, 5, 6, 7])
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_exchange_clipped_blocks_excludes_nontransfer_groups():
|
||||
"""Scheduler block tables must match the worker's registered regions."""
|
||||
sched = make_nixl_scheduler()
|
||||
sched.kv_cache_config = make_kv_cache_config(block_size=16, swa_enabled=True)
|
||||
sched.kv_cache_config.kv_cache_groups[1].enable_kv_transfer = False
|
||||
|
||||
assert sched.get_exchange_clipped_blocks(([1, 2], [9])) == ([1, 2],)
|
||||
|
||||
|
||||
# ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ─────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -524,6 +524,10 @@ def make_nixl_scheduler(
|
||||
sched = object.__new__(NixlConnectorScheduler)
|
||||
sched._has_mamba = has_mamba
|
||||
sched._is_hma_required = is_hma_required
|
||||
sched.kv_cache_config = make_kv_cache_config(
|
||||
block_size=16,
|
||||
mamba_enabled=has_mamba,
|
||||
)
|
||||
|
||||
if heartbeat:
|
||||
sched._heartbeat_by_engine = {}
|
||||
@@ -582,6 +586,10 @@ def make_nixl_push_scheduler(
|
||||
sched.side_channel_port = 5600
|
||||
sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled
|
||||
sched._has_mamba = has_mamba
|
||||
sched.kv_cache_config = make_kv_cache_config(
|
||||
block_size=16,
|
||||
mamba_enabled=has_mamba,
|
||||
)
|
||||
|
||||
# vllm_config is consulted for parallel_config.tensor_parallel_size.
|
||||
vllm_config = MagicMock()
|
||||
|
||||
Reference in New Issue
Block a user