Keep scheduler alive for delayed KV connector frees (#43433)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
This commit is contained in:
@@ -1882,7 +1882,16 @@ class Scheduler(SchedulerInterface):
|
||||
return num_waiting + len(self.running)
|
||||
|
||||
def has_finished_requests(self) -> bool:
|
||||
return len(self.finished_req_ids) > 0
|
||||
if self.finished_req_ids:
|
||||
return True
|
||||
if self.connector is None:
|
||||
return False
|
||||
# Finished requests waiting on delayed connector cleanup remain in
|
||||
# self.requests after they have been removed from scheduling queues.
|
||||
num_in_queues = (
|
||||
len(self.waiting) + len(self.skipped_waiting) + len(self.running)
|
||||
)
|
||||
return len(self.requests) > num_in_queues
|
||||
|
||||
def reset_prefix_cache(
|
||||
self, reset_running_requests: bool = False, reset_connector: bool = False
|
||||
|
||||
@@ -1242,11 +1242,10 @@ class EngineCoreProc(EngineCore):
|
||||
# Post-step hook.
|
||||
self.post_step(model_executed)
|
||||
|
||||
# If no model execution happened but there are waiting requests
|
||||
# (e.g., WAITING_FOR_REMOTE_KVS), yield the GIL briefly to allow
|
||||
# background threads (like NIXL handshake) to make progress.
|
||||
# Without this, the tight polling loop can starve background threads.
|
||||
if not model_executed and self.scheduler.has_unfinished_requests():
|
||||
# If no model execution happened but there is still scheduler work
|
||||
# (e.g. WAITING_FOR_REMOTE_KVS or delayed KV connector frees), yield
|
||||
# the GIL briefly to allow background transfer threads to make progress.
|
||||
if not model_executed and self.scheduler.has_requests():
|
||||
time.sleep(0.001)
|
||||
|
||||
return model_executed
|
||||
|
||||
Reference in New Issue
Block a user