[EC Connector] Non blocking EC Connector lookup (#41627)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
omerpaz95
2026-06-02 08:48:25 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7c37096620
commit d247a9dc13
3 changed files with 154 additions and 0 deletions
@@ -211,6 +211,23 @@ class ECConnectorBase(ABC):
"""
pass
def ensure_cache_available(
self, request: "Request", num_computed_tokens: int
) -> bool:
"""
Ensure encoder cache items are available for the given request.
May initiate asynchronous transfers for items not yet local.
Args:
request: the request whose multimodal features to check.
num_computed_tokens: tokens already covered by cached KV blocks.
Returns:
True if all items are ready or no transfer is needed.
False if any items are still in transit (request should be deferred).
"""
return True
@abstractmethod
def update_state_after_alloc(self, request: "Request", index: int):
"""