Keep scheduler alive for delayed KV connector frees (#43433)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
This commit is contained in:
@@ -3928,6 +3928,45 @@ def test_abort_request_finished_recving():
|
||||
assert not scheduler.finished_recving_kv_req_ids
|
||||
|
||||
|
||||
def test_delayed_kv_connector_free_keeps_scheduler_active():
|
||||
scheduler = create_scheduler(use_kv_connector=True)
|
||||
queued_request, request = create_requests(
|
||||
num_requests=2, req_ids=["queued", "finished"]
|
||||
)
|
||||
scheduler.add_request(queued_request)
|
||||
|
||||
assert not scheduler.has_finished_requests()
|
||||
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
scheduler.requests[request.request_id] = request
|
||||
scheduler.finished_req_ids = set()
|
||||
|
||||
assert scheduler.has_finished_requests()
|
||||
assert scheduler.has_requests()
|
||||
|
||||
scheduler_output = SchedulerOutput(
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=CachedRequestData.make_empty(),
|
||||
num_scheduled_tokens={},
|
||||
total_num_scheduled_tokens=0,
|
||||
scheduled_encoder_inputs={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
model_runner_output = ModelRunnerOutput(
|
||||
req_ids=[],
|
||||
req_id_to_index={},
|
||||
kv_connector_output=KVConnectorOutput(finished_sending={request.request_id}),
|
||||
)
|
||||
|
||||
scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
|
||||
assert request.request_id not in scheduler.requests
|
||||
assert not scheduler.has_finished_requests()
|
||||
|
||||
|
||||
# ==============================================================================
|
||||
# Variable-length encoder cross-attention block allocation tests
|
||||
# ==============================================================================
|
||||
|
||||
@@ -258,10 +258,12 @@ def test_multi_example_connector_consistency():
|
||||
)
|
||||
|
||||
events = get_connector_events()
|
||||
storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"])
|
||||
storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"])
|
||||
# First event is bind_gpu_block_pool from initialization, then
|
||||
# set_xfer_handshake_metadata, then on_new_request when the request is enqueued,
|
||||
# then get_num_new_matched_tokens and update_state_after_alloc from generate().
|
||||
assert events["storage1-SCHEDULER"][:6] == [
|
||||
assert storage1_scheduler_events[:6] == [
|
||||
"bind_gpu_block_pool",
|
||||
"set_xfer_handshake_metadata",
|
||||
"on_new_request",
|
||||
@@ -281,7 +283,7 @@ def test_multi_example_connector_consistency():
|
||||
"wait_for_layer_load",
|
||||
"save_kv_layer",
|
||||
]
|
||||
assert events["storage2-SCHEDULER"][:6] == [
|
||||
assert storage2_scheduler_events[:6] == [
|
||||
"bind_gpu_block_pool",
|
||||
"set_xfer_handshake_metadata",
|
||||
"on_new_request",
|
||||
@@ -312,13 +314,15 @@ def test_multi_example_connector_consistency():
|
||||
# connector so update_state_after_alloc will be with allocated blocks
|
||||
# on that one but with zero blocks for others (first nonzero match is
|
||||
# chosen).
|
||||
assert events["storage1-SCHEDULER"][:4] == [
|
||||
storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"])
|
||||
storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"])
|
||||
assert storage1_scheduler_events[:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[7] 96",
|
||||
"build_connector_meta",
|
||||
]
|
||||
assert events["storage2-SCHEDULER"][:4] == [
|
||||
assert storage2_scheduler_events[:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[0] 0",
|
||||
@@ -340,13 +344,15 @@ def test_multi_example_connector_consistency():
|
||||
# return 0 from the first connector, but the second connector should have
|
||||
# a hit, so update_state_after_alloc will only be called with allocated
|
||||
# blocks for the second connector.
|
||||
assert events["storage1-SCHEDULER"][:4] == [
|
||||
storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"])
|
||||
storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"])
|
||||
assert storage1_scheduler_events[:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[0] 0",
|
||||
"build_connector_meta",
|
||||
]
|
||||
assert events["storage2-SCHEDULER"][:4] == [
|
||||
assert storage2_scheduler_events[:4] == [
|
||||
"on_new_request",
|
||||
"get_num_new_matched_tokens 0",
|
||||
"update_state_after_alloc num_blocks=[7] 96",
|
||||
@@ -358,6 +364,10 @@ def test_multi_example_connector_consistency():
|
||||
shutil.rmtree(storage_2_path)
|
||||
|
||||
|
||||
def _ignore_event_collection(events: list[str]) -> list[str]:
|
||||
return [event for event in events if event != "take_events"]
|
||||
|
||||
|
||||
def get_connector_events() -> dict[str, list[str]]:
|
||||
# Read in connector events and reset the files.
|
||||
import glob
|
||||
|
||||
@@ -1882,7 +1882,16 @@ class Scheduler(SchedulerInterface):
|
||||
return num_waiting + len(self.running)
|
||||
|
||||
def has_finished_requests(self) -> bool:
|
||||
return len(self.finished_req_ids) > 0
|
||||
if self.finished_req_ids:
|
||||
return True
|
||||
if self.connector is None:
|
||||
return False
|
||||
# Finished requests waiting on delayed connector cleanup remain in
|
||||
# self.requests after they have been removed from scheduling queues.
|
||||
num_in_queues = (
|
||||
len(self.waiting) + len(self.skipped_waiting) + len(self.running)
|
||||
)
|
||||
return len(self.requests) > num_in_queues
|
||||
|
||||
def reset_prefix_cache(
|
||||
self, reset_running_requests: bool = False, reset_connector: bool = False
|
||||
|
||||
@@ -1242,11 +1242,10 @@ class EngineCoreProc(EngineCore):
|
||||
# Post-step hook.
|
||||
self.post_step(model_executed)
|
||||
|
||||
# If no model execution happened but there are waiting requests
|
||||
# (e.g., WAITING_FOR_REMOTE_KVS), yield the GIL briefly to allow
|
||||
# background threads (like NIXL handshake) to make progress.
|
||||
# Without this, the tight polling loop can starve background threads.
|
||||
if not model_executed and self.scheduler.has_unfinished_requests():
|
||||
# If no model execution happened but there is still scheduler work
|
||||
# (e.g. WAITING_FOR_REMOTE_KVS or delayed KV connector frees), yield
|
||||
# the GIL briefly to allow background transfer threads to make progress.
|
||||
if not model_executed and self.scheduler.has_requests():
|
||||
time.sleep(0.001)
|
||||
|
||||
return model_executed
|
||||
|
||||
Reference in New Issue
Block a user