Keep scheduler alive for delayed KV connector frees (#43433)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
This commit is contained in:
Gabriel Wu
2026-05-23 06:23:32 +00:00
committed by GitHub
parent 09a219c075
commit 82536acc54
4 changed files with 69 additions and 12 deletions
+10 -1
View File
@@ -1882,7 +1882,16 @@ class Scheduler(SchedulerInterface):
return num_waiting + len(self.running)
def has_finished_requests(self) -> bool:
return len(self.finished_req_ids) > 0
if self.finished_req_ids:
return True
if self.connector is None:
return False
# Finished requests waiting on delayed connector cleanup remain in
# self.requests after they have been removed from scheduling queues.
num_in_queues = (
len(self.waiting) + len(self.skipped_waiting) + len(self.running)
)
return len(self.requests) > num_in_queues
def reset_prefix_cache(
self, reset_running_requests: bool = False, reset_connector: bool = False
+4 -5
View File
@@ -1242,11 +1242,10 @@ class EngineCoreProc(EngineCore):
# Post-step hook.
self.post_step(model_executed)
# If no model execution happened but there are waiting requests
# (e.g., WAITING_FOR_REMOTE_KVS), yield the GIL briefly to allow
# background threads (like NIXL handshake) to make progress.
# Without this, the tight polling loop can starve background threads.
if not model_executed and self.scheduler.has_unfinished_requests():
# If no model execution happened but there is still scheduler work
# (e.g. WAITING_FOR_REMOTE_KVS or delayed KV connector frees), yield
# the GIL briefly to allow background transfer threads to make progress.
if not model_executed and self.scheduler.has_requests():
time.sleep(0.001)
return model_executed