[BugFix] Scope divergent hybrid cache hits to capable connectors (#50344)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Yifan Qiao
2026-08-09 15:38:13 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 83ad767eed
commit d6941300fc
8 changed files with 130 additions and 27 deletions
+18 -20
View File
@@ -423,6 +423,18 @@ class Scheduler(SchedulerInterface):
end = min((s for s in stops if start < s < end), default=end)
return max(end - start, 0)
def _get_local_prefix_cache_hit(
self, request: Request
) -> tuple[KVCacheBlocks, int, int, bool]:
connector = self.connector
if connector is not None and connector.supports_divergent_local_hybrid_hits:
return self.kv_cache_manager.get_computed_blocks_for_connector(request)
blocks, num_local, shared_prefix_boundary = (
self.kv_cache_manager.get_computed_blocks(request)
)
return blocks, num_local, shared_prefix_boundary, False
def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput:
self.current_step += 1
# NOTE(woosuk) on the scheduling algorithm:
@@ -739,26 +751,12 @@ class Scheduler(SchedulerInterface):
# Get already-cached tokens.
if request.num_computed_tokens == 0:
did_prefix_cache_lookup = True
hit_diverged = False
# Get locally-cached tokens.
if self.connector is not None:
# A KV connector transfers the missing suffix, which needs a
# hybrid-aware lookup that can diverge across groups.
(
new_computed_blocks,
num_new_local_computed_tokens,
request.shared_prefix_boundary,
hit_diverged,
) = self.kv_cache_manager.get_computed_blocks_for_connector(
request
)
else:
(
new_computed_blocks,
num_new_local_computed_tokens,
# Marconi shared-prefix junction to pin; 0 if none.
request.shared_prefix_boundary,
) = self.kv_cache_manager.get_computed_blocks(request)
(
new_computed_blocks,
num_new_local_computed_tokens,
request.shared_prefix_boundary,
hit_diverged,
) = self._get_local_prefix_cache_hit(request)
# Get externally-cached tokens if using a KVConnector.
if self.connector is not None: