Keep scheduler alive for delayed KV connector frees (#43433)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
This commit is contained in:
Gabriel Wu
2026-05-23 06:23:32 +00:00
committed by GitHub
parent 09a219c075
commit 82536acc54
4 changed files with 69 additions and 12 deletions
+39
View File
@@ -3928,6 +3928,45 @@ def test_abort_request_finished_recving():
assert not scheduler.finished_recving_kv_req_ids
def test_delayed_kv_connector_free_keeps_scheduler_active():
scheduler = create_scheduler(use_kv_connector=True)
queued_request, request = create_requests(
num_requests=2, req_ids=["queued", "finished"]
)
scheduler.add_request(queued_request)
assert not scheduler.has_finished_requests()
request.status = RequestStatus.FINISHED_STOPPED
scheduler.requests[request.request_id] = request
scheduler.finished_req_ids = set()
assert scheduler.has_finished_requests()
assert scheduler.has_requests()
scheduler_output = SchedulerOutput(
scheduled_new_reqs=[],
scheduled_cached_reqs=CachedRequestData.make_empty(),
num_scheduled_tokens={},
total_num_scheduled_tokens=0,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
model_runner_output = ModelRunnerOutput(
req_ids=[],
req_id_to_index={},
kv_connector_output=KVConnectorOutput(finished_sending={request.request_id}),
)
scheduler.update_from_output(scheduler_output, model_runner_output)
assert request.request_id not in scheduler.requests
assert not scheduler.has_finished_requests()
# ==============================================================================
# Variable-length encoder cross-attention block allocation tests
# ==============================================================================
@@ -258,10 +258,12 @@ def test_multi_example_connector_consistency():
)
events = get_connector_events()
storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"])
storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"])
# First event is bind_gpu_block_pool from initialization, then
# set_xfer_handshake_metadata, then on_new_request when the request is enqueued,
# then get_num_new_matched_tokens and update_state_after_alloc from generate().
assert events["storage1-SCHEDULER"][:6] == [
assert storage1_scheduler_events[:6] == [
"bind_gpu_block_pool",
"set_xfer_handshake_metadata",
"on_new_request",
@@ -281,7 +283,7 @@ def test_multi_example_connector_consistency():
"wait_for_layer_load",
"save_kv_layer",
]
assert events["storage2-SCHEDULER"][:6] == [
assert storage2_scheduler_events[:6] == [
"bind_gpu_block_pool",
"set_xfer_handshake_metadata",
"on_new_request",
@@ -312,13 +314,15 @@ def test_multi_example_connector_consistency():
# connector so update_state_after_alloc will be with allocated blocks
# on that one but with zero blocks for others (first nonzero match is
# chosen).
assert events["storage1-SCHEDULER"][:4] == [
storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"])
storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"])
assert storage1_scheduler_events[:4] == [
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[7] 96",
"build_connector_meta",
]
assert events["storage2-SCHEDULER"][:4] == [
assert storage2_scheduler_events[:4] == [
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[0] 0",
@@ -340,13 +344,15 @@ def test_multi_example_connector_consistency():
# return 0 from the first connector, but the second connector should have
# a hit, so update_state_after_alloc will only be called with allocated
# blocks for the second connector.
assert events["storage1-SCHEDULER"][:4] == [
storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"])
storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"])
assert storage1_scheduler_events[:4] == [
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[0] 0",
"build_connector_meta",
]
assert events["storage2-SCHEDULER"][:4] == [
assert storage2_scheduler_events[:4] == [
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[7] 96",
@@ -358,6 +364,10 @@ def test_multi_example_connector_consistency():
shutil.rmtree(storage_2_path)
def _ignore_event_collection(events: list[str]) -> list[str]:
return [event for event in events if event != "take_events"]
def get_connector_events() -> dict[str, list[str]]:
# Read in connector events and reset the files.
import glob
+10 -1
View File
@@ -1882,7 +1882,16 @@ class Scheduler(SchedulerInterface):
return num_waiting + len(self.running)
def has_finished_requests(self) -> bool:
return len(self.finished_req_ids) > 0
if self.finished_req_ids:
return True
if self.connector is None:
return False
# Finished requests waiting on delayed connector cleanup remain in
# self.requests after they have been removed from scheduling queues.
num_in_queues = (
len(self.waiting) + len(self.skipped_waiting) + len(self.running)
)
return len(self.requests) > num_in_queues
def reset_prefix_cache(
self, reset_running_requests: bool = False, reset_connector: bool = False
+4 -5
View File
@@ -1242,11 +1242,10 @@ class EngineCoreProc(EngineCore):
# Post-step hook.
self.post_step(model_executed)
# If no model execution happened but there are waiting requests
# (e.g., WAITING_FOR_REMOTE_KVS), yield the GIL briefly to allow
# background threads (like NIXL handshake) to make progress.
# Without this, the tight polling loop can starve background threads.
if not model_executed and self.scheduler.has_unfinished_requests():
# If no model execution happened but there is still scheduler work
# (e.g. WAITING_FOR_REMOTE_KVS or delayed KV connector frees), yield
# the GIL briefly to allow background transfer threads to make progress.
if not model_executed and self.scheduler.has_requests():
time.sleep(0.001)
return model_executed