[BugFix] Scope divergent hybrid cache hits to capable connectors (#50344)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
83ad767eed
commit
d6941300fc
@@ -423,6 +423,18 @@ class Scheduler(SchedulerInterface):
|
||||
end = min((s for s in stops if start < s < end), default=end)
|
||||
return max(end - start, 0)
|
||||
|
||||
def _get_local_prefix_cache_hit(
|
||||
self, request: Request
|
||||
) -> tuple[KVCacheBlocks, int, int, bool]:
|
||||
connector = self.connector
|
||||
if connector is not None and connector.supports_divergent_local_hybrid_hits:
|
||||
return self.kv_cache_manager.get_computed_blocks_for_connector(request)
|
||||
|
||||
blocks, num_local, shared_prefix_boundary = (
|
||||
self.kv_cache_manager.get_computed_blocks(request)
|
||||
)
|
||||
return blocks, num_local, shared_prefix_boundary, False
|
||||
|
||||
def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput:
|
||||
self.current_step += 1
|
||||
# NOTE(woosuk) on the scheduling algorithm:
|
||||
@@ -739,26 +751,12 @@ class Scheduler(SchedulerInterface):
|
||||
# Get already-cached tokens.
|
||||
if request.num_computed_tokens == 0:
|
||||
did_prefix_cache_lookup = True
|
||||
hit_diverged = False
|
||||
# Get locally-cached tokens.
|
||||
if self.connector is not None:
|
||||
# A KV connector transfers the missing suffix, which needs a
|
||||
# hybrid-aware lookup that can diverge across groups.
|
||||
(
|
||||
new_computed_blocks,
|
||||
num_new_local_computed_tokens,
|
||||
request.shared_prefix_boundary,
|
||||
hit_diverged,
|
||||
) = self.kv_cache_manager.get_computed_blocks_for_connector(
|
||||
request
|
||||
)
|
||||
else:
|
||||
(
|
||||
new_computed_blocks,
|
||||
num_new_local_computed_tokens,
|
||||
# Marconi shared-prefix junction to pin; 0 if none.
|
||||
request.shared_prefix_boundary,
|
||||
) = self.kv_cache_manager.get_computed_blocks(request)
|
||||
(
|
||||
new_computed_blocks,
|
||||
num_new_local_computed_tokens,
|
||||
request.shared_prefix_boundary,
|
||||
hit_diverged,
|
||||
) = self._get_local_prefix_cache_hit(request)
|
||||
|
||||
# Get externally-cached tokens if using a KVConnector.
|
||||
if self.connector is not None:
|
||||
|
||||
Reference in New Issue
Block a user