[Bugfix][Core] Exempt exactly the blocks an async KV load writes from zeroing (#59504)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Codex
parent
1a001d5842
commit
ca0df1f122
@@ -482,6 +482,16 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
|
||||
"""KV cache groups restored by this connector's load for ``request``.
|
||||
|
||||
Called after ``get_num_new_matched_tokens`` returned a positive count.
|
||||
Defaults to the prefix-cacheable groups, all a hash-addressed store
|
||||
holds; a connector transferring the request's own blocks restores
|
||||
every transfer group.
|
||||
"""
|
||||
return self._kv_cache_config.prefix_cacheable_group_ids
|
||||
|
||||
@abstractmethod
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
|
||||
@@ -748,6 +748,9 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# Scheduler Side Methods
|
||||
############################################################
|
||||
|
||||
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
|
||||
return self._kv_cache_config.transfer_group_ids
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int, bool]:
|
||||
|
||||
@@ -426,6 +426,10 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# ==============================
|
||||
# Scheduler-side methods
|
||||
# ==============================
|
||||
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
|
||||
connector = self._connectors[self._requests_to_connector[request.request_id]]
|
||||
return connector.get_loaded_kv_cache_group_ids(request)
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: "Request",
|
||||
|
||||
@@ -164,6 +164,9 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def get_finished_count(self) -> int | None:
|
||||
return None
|
||||
|
||||
def get_loaded_kv_cache_group_ids(self, request: "Request") -> tuple[int, ...]:
|
||||
return self._kv_cache_config.transfer_group_ids
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
|
||||
Reference in New Issue
Block a user