[Hybrid] Marconi-style admission policy for hybrid cache (#37898)

Signed-off-by: Stanislaw Wozniak <stw@zurich.ibm.com>
This commit is contained in:
Stan Wozniak
2026-06-10 10:03:13 -07:00
committed by GitHub
parent 2ba68d9bf7
commit dc66e01a70
3 changed files with 89 additions and 0 deletions
+6
View File
@@ -611,6 +611,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
num_groups = len(self.kv_cache_config.kv_cache_groups)
hit_length = max_cache_hit_length
longest_hit_length = 0
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
# Simple hybrid (1 full attn + 1 other): one iteration suffices.
@@ -667,6 +668,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
for group_id, blocks in zip(group_ids, hit_blocks):
hit_blocks_by_group[group_id] = blocks
longest_hit_length = max(longest_hit_length, curr_hit_length)
if curr_hit_length >= hit_length:
break
hit_length = curr_hit_length
@@ -681,6 +684,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
if (blks := hit_blocks_by_group[group_id]) is not None:
del blks[num_blocks:]
# Uncached shared prefix detection: If any attn. group cached a longer prefix
# than the current prefix, it is an uncached common prefix across requests:
self.num_uncached_common_prefix_tokens = longest_hit_length - hit_length
return tuple(
blocks if blocks is not None else [] for blocks in hit_blocks_by_group
), hit_length
+21
View File
@@ -296,6 +296,7 @@ class Scheduler(SchedulerInterface):
num_new_tokens: int,
num_new_local_computed_tokens: int = 0,
num_external_computed_tokens: int = 0,
num_uncached_common_prefix_tokens: int = 0,
) -> int:
num_computed_tokens = (
request.num_computed_tokens
@@ -335,6 +336,16 @@ class Scheduler(SchedulerInterface):
else:
# prefill the last few tokens
pass
# Marconi cache admission optimization:
# cache common prefixes by scheduling num_new_tokens = common prefix length
if (
num_uncached_common_prefix_tokens >= block_size
and num_new_tokens > num_uncached_common_prefix_tokens
):
num_new_tokens = num_uncached_common_prefix_tokens
# keep alignment to block_size
num_new_tokens = num_new_tokens // block_size * block_size
return num_new_tokens
def schedule(self) -> SchedulerOutput:
@@ -604,6 +615,7 @@ class Scheduler(SchedulerInterface):
num_external_computed_tokens = 0
load_kv_async = False
connector_prefix_cache_queries, connector_prefix_cache_hits = 0, 0
num_uncached_common_prefix_tokens = 0
# Get already-cached tokens.
if request.num_computed_tokens == 0:
@@ -612,6 +624,14 @@ class Scheduler(SchedulerInterface):
self.kv_cache_manager.get_computed_blocks(request)
)
# In case of hybrid models, obtain hint for Marconi-style APC logic
if self.has_mamba_layers:
num_uncached_common_prefix_tokens = getattr(
self.kv_cache_manager.coordinator,
"num_uncached_common_prefix_tokens",
0,
)
# Get externally-cached tokens if using a KVConnector.
if self.connector is not None:
ext_tokens, load_kv_async = (
@@ -724,6 +744,7 @@ class Scheduler(SchedulerInterface):
num_new_tokens,
num_new_local_computed_tokens,
num_external_computed_tokens,
num_uncached_common_prefix_tokens,
)
if num_new_tokens == 0:
break