[EC Connector] Non blocking EC Connector lookup (#41627)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
omerpaz95
2026-06-02 08:48:25 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7c37096620
commit d247a9dc13
3 changed files with 154 additions and 0 deletions
@@ -211,6 +211,23 @@ class ECConnectorBase(ABC):
"""
pass
def ensure_cache_available(
self, request: "Request", num_computed_tokens: int
) -> bool:
"""
Ensure encoder cache items are available for the given request.
May initiate asynchronous transfers for items not yet local.
Args:
request: the request whose multimodal features to check.
num_computed_tokens: tokens already covered by cached KV blocks.
Returns:
True if all items are ready or no transfer is needed.
False if any items are still in transit (request should be deferred).
"""
return True
@abstractmethod
def update_state_after_alloc(self, request: "Request", index: int):
"""
+12
View File
@@ -629,6 +629,18 @@ class Scheduler(SchedulerInterface):
)
assert num_computed_tokens <= request.num_tokens
# Skip request with pending mm encoding prefetches
if (
self.ec_connector is not None
and request.mm_features
and not self.ec_connector.ensure_cache_available(
request, num_computed_tokens
)
):
request_queue.pop_request()
step_skipped_waiting.prepend_request(request)
continue
# Track first scheduled prefill, not post-preemption repeat prefills
if request.prefill_stats is not None:
assert num_computed_tokens <= request.num_prompt_tokens