[Hybrid] Marconi-style admission policy for hybrid cache (#37898)
Signed-off-by: Stanislaw Wozniak <stw@zurich.ibm.com>
This commit is contained in:
@@ -611,6 +611,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
|
||||
num_groups = len(self.kv_cache_config.kv_cache_groups)
|
||||
hit_length = max_cache_hit_length
|
||||
longest_hit_length = 0
|
||||
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
|
||||
|
||||
# Simple hybrid (1 full attn + 1 other): one iteration suffices.
|
||||
@@ -667,6 +668,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
for group_id, blocks in zip(group_ids, hit_blocks):
|
||||
hit_blocks_by_group[group_id] = blocks
|
||||
|
||||
longest_hit_length = max(longest_hit_length, curr_hit_length)
|
||||
|
||||
if curr_hit_length >= hit_length:
|
||||
break
|
||||
hit_length = curr_hit_length
|
||||
@@ -681,6 +684,9 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
if (blks := hit_blocks_by_group[group_id]) is not None:
|
||||
del blks[num_blocks:]
|
||||
|
||||
# Uncached shared prefix detection: If any attn. group cached a longer prefix
|
||||
# than the current prefix, it is an uncached common prefix across requests:
|
||||
self.num_uncached_common_prefix_tokens = longest_hit_length - hit_length
|
||||
return tuple(
|
||||
blocks if blocks is not None else [] for blocks in hit_blocks_by_group
|
||||
), hit_length
|
||||
|
||||
@@ -296,6 +296,7 @@ class Scheduler(SchedulerInterface):
|
||||
num_new_tokens: int,
|
||||
num_new_local_computed_tokens: int = 0,
|
||||
num_external_computed_tokens: int = 0,
|
||||
num_uncached_common_prefix_tokens: int = 0,
|
||||
) -> int:
|
||||
num_computed_tokens = (
|
||||
request.num_computed_tokens
|
||||
@@ -335,6 +336,16 @@ class Scheduler(SchedulerInterface):
|
||||
else:
|
||||
# prefill the last few tokens
|
||||
pass
|
||||
|
||||
# Marconi cache admission optimization:
|
||||
# cache common prefixes by scheduling num_new_tokens = common prefix length
|
||||
if (
|
||||
num_uncached_common_prefix_tokens >= block_size
|
||||
and num_new_tokens > num_uncached_common_prefix_tokens
|
||||
):
|
||||
num_new_tokens = num_uncached_common_prefix_tokens
|
||||
# keep alignment to block_size
|
||||
num_new_tokens = num_new_tokens // block_size * block_size
|
||||
return num_new_tokens
|
||||
|
||||
def schedule(self) -> SchedulerOutput:
|
||||
@@ -604,6 +615,7 @@ class Scheduler(SchedulerInterface):
|
||||
num_external_computed_tokens = 0
|
||||
load_kv_async = False
|
||||
connector_prefix_cache_queries, connector_prefix_cache_hits = 0, 0
|
||||
num_uncached_common_prefix_tokens = 0
|
||||
|
||||
# Get already-cached tokens.
|
||||
if request.num_computed_tokens == 0:
|
||||
@@ -612,6 +624,14 @@ class Scheduler(SchedulerInterface):
|
||||
self.kv_cache_manager.get_computed_blocks(request)
|
||||
)
|
||||
|
||||
# In case of hybrid models, obtain hint for Marconi-style APC logic
|
||||
if self.has_mamba_layers:
|
||||
num_uncached_common_prefix_tokens = getattr(
|
||||
self.kv_cache_manager.coordinator,
|
||||
"num_uncached_common_prefix_tokens",
|
||||
0,
|
||||
)
|
||||
|
||||
# Get externally-cached tokens if using a KVConnector.
|
||||
if self.connector is not None:
|
||||
ext_tokens, load_kv_async = (
|
||||
@@ -724,6 +744,7 @@ class Scheduler(SchedulerInterface):
|
||||
num_new_tokens,
|
||||
num_new_local_computed_tokens,
|
||||
num_external_computed_tokens,
|
||||
num_uncached_common_prefix_tokens,
|
||||
)
|
||||
if num_new_tokens == 0:
|
||||
break
|
||||
|
||||
Reference in New Issue
Block a user