[Bugfix][Core] Exempt exactly the blocks an async KV load writes from zeroing (#59504)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Yifan Qiao
2026-10-03 06:26:12 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Codex
parent 1a001d5842
commit ca0df1f122
17 changed files with 162 additions and 72 deletions
@@ -482,6 +482,16 @@ class KVConnectorBase_V1(ABC):
"""
return
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
"""KV cache groups restored by this connector's load for ``request``.
Called after ``get_num_new_matched_tokens`` returned a positive count.
Defaults to the prefix-cacheable groups, all a hash-addressed store
holds; a connector transferring the request's own blocks restores
every transfer group.
"""
return self._kv_cache_config.prefix_cacheable_group_ids
@abstractmethod
def get_num_new_matched_tokens(
self,
@@ -748,6 +748,9 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
# Scheduler Side Methods
############################################################
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
return self._kv_cache_config.transfer_group_ids
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int, bool]:
@@ -426,6 +426,10 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
# ==============================
# Scheduler-side methods
# ==============================
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
connector = self._connectors[self._requests_to_connector[request.request_id]]
return connector.get_loaded_kv_cache_group_ids(request)
def get_num_new_matched_tokens(
self,
request: "Request",
@@ -164,6 +164,9 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
def get_finished_count(self) -> int | None:
return None
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
return self._kv_cache_config.transfer_group_ids
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int | None, bool]: