Keep scheduler alive for delayed KV connector frees (#43433)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
This commit is contained in:
Gabriel Wu
2026-05-23 06:23:32 +00:00
committed by GitHub
parent 09a219c075
commit 82536acc54
4 changed files with 69 additions and 12 deletions
+39
View File
@@ -3928,6 +3928,45 @@ def test_abort_request_finished_recving():
assert not scheduler.finished_recving_kv_req_ids
def test_delayed_kv_connector_free_keeps_scheduler_active():
scheduler = create_scheduler(use_kv_connector=True)
queued_request, request = create_requests(
num_requests=2, req_ids=["queued", "finished"]
)
scheduler.add_request(queued_request)
assert not scheduler.has_finished_requests()
request.status = RequestStatus.FINISHED_STOPPED
scheduler.requests[request.request_id] = request
scheduler.finished_req_ids = set()
assert scheduler.has_finished_requests()
assert scheduler.has_requests()
scheduler_output = SchedulerOutput(
scheduled_new_reqs=[],
scheduled_cached_reqs=CachedRequestData.make_empty(),
num_scheduled_tokens={},
total_num_scheduled_tokens=0,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
model_runner_output = ModelRunnerOutput(
req_ids=[],
req_id_to_index={},
kv_connector_output=KVConnectorOutput(finished_sending={request.request_id}),
)
scheduler.update_from_output(scheduler_output, model_runner_output)
assert request.request_id not in scheduler.requests
assert not scheduler.has_finished_requests()
# ==============================================================================
# Variable-length encoder cross-attention block allocation tests
# ==============================================================================