[BugFix] Scope divergent hybrid cache hits to capable connectors (#50344)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
83ad767eed
commit
d6941300fc
@@ -173,6 +173,15 @@ class KVConnectorBase_V1(ABC):
|
||||
Base class for KV connectors.
|
||||
"""
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
"""Whether external hits can complete divergent local hybrid hits.
|
||||
|
||||
A capable connector restores lagging recurrent state when the local
|
||||
full-attention group reaches a deeper boundary. Defaults to False.
|
||||
"""
|
||||
return False
|
||||
|
||||
@property
|
||||
def prefer_cross_layer_blocks(self) -> bool:
|
||||
"""
|
||||
|
||||
@@ -203,6 +203,12 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# Propagated from scheduler to worker side via the connector metadata.
|
||||
self._extra_async_saves: dict[str, int] = {}
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
return bool(self._connectors) and all(
|
||||
c.supports_divergent_local_hybrid_hits for c in self._connectors
|
||||
)
|
||||
|
||||
@property
|
||||
def prefer_cross_layer_blocks(self) -> bool:
|
||||
if not self._connectors:
|
||||
|
||||
@@ -79,6 +79,10 @@ logger = init_logger(__name__)
|
||||
class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""Base connector with common logic shared by pull and push modes."""
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def prefer_cross_layer_blocks(self) -> bool:
|
||||
if any(
|
||||
|
||||
Reference in New Issue
Block a user