[EC Connector] Non blocking EC Connector lookup (#41627)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7c37096620
commit
d247a9dc13
@@ -211,6 +211,23 @@ class ECConnectorBase(ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
def ensure_cache_available(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
"""
|
||||
Ensure encoder cache items are available for the given request.
|
||||
May initiate asynchronous transfers for items not yet local.
|
||||
|
||||
Args:
|
||||
request: the request whose multimodal features to check.
|
||||
num_computed_tokens: tokens already covered by cached KV blocks.
|
||||
|
||||
Returns:
|
||||
True if all items are ready or no transfer is needed.
|
||||
False if any items are still in transit (request should be deferred).
|
||||
"""
|
||||
return True
|
||||
|
||||
@abstractmethod
|
||||
def update_state_after_alloc(self, request: "Request", index: int):
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user