[EC Connector] Non blocking EC Connector lookup (#41627)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7c37096620
commit
d247a9dc13
@@ -211,6 +211,23 @@ class ECConnectorBase(ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
def ensure_cache_available(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
"""
|
||||
Ensure encoder cache items are available for the given request.
|
||||
May initiate asynchronous transfers for items not yet local.
|
||||
|
||||
Args:
|
||||
request: the request whose multimodal features to check.
|
||||
num_computed_tokens: tokens already covered by cached KV blocks.
|
||||
|
||||
Returns:
|
||||
True if all items are ready or no transfer is needed.
|
||||
False if any items are still in transit (request should be deferred).
|
||||
"""
|
||||
return True
|
||||
|
||||
@abstractmethod
|
||||
def update_state_after_alloc(self, request: "Request", index: int):
|
||||
"""
|
||||
|
||||
@@ -629,6 +629,18 @@ class Scheduler(SchedulerInterface):
|
||||
)
|
||||
assert num_computed_tokens <= request.num_tokens
|
||||
|
||||
# Skip request with pending mm encoding prefetches
|
||||
if (
|
||||
self.ec_connector is not None
|
||||
and request.mm_features
|
||||
and not self.ec_connector.ensure_cache_available(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
):
|
||||
request_queue.pop_request()
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
continue
|
||||
|
||||
# Track first scheduled prefill, not post-preemption repeat prefills
|
||||
if request.prefill_stats is not None:
|
||||
assert num_computed_tokens <= request.num_prompt_tokens
|
||||
|
||||
Reference in New Issue
Block a user