From 42dbe4578c7bbeceadc28bd8c6a3826949163f5c Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Mon, 28 Sep 2026 13:03:16 -0700 Subject: [PATCH] [Core] Rework scheduler `skipped_waiting` queue (#58947) Signed-off-by: Nick Hill Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> --- .../engine-core-client/src/protocol/stats.rs | 4 +- tests/v1/core/test_scheduler.py | 60 +++--- tests/v1/core/test_worker_slot_overflow.py | 4 +- .../unit/test_error_propagation.py | 5 +- .../unit/test_invalid_blocks_correctness.py | 2 +- .../unit/test_kv_load_failure_recovery.py | 18 +- .../kv_connector/unit/test_nixl_connector.py | 2 +- .../unit/test_remote_prefill_lifecycle.py | 72 ++++++- tests/v1/kv_connector/unit/utils.py | 2 + .../test_scheduler_streaming.py | 6 +- vllm/v1/core/sched/request_queue.py | 6 +- vllm/v1/core/sched/scheduler.py | 183 +++++++++--------- vllm/v1/metrics/stats.py | 5 +- 13 files changed, 220 insertions(+), 149 deletions(-) diff --git a/rust/src/engine-core-client/src/protocol/stats.rs b/rust/src/engine-core-client/src/protocol/stats.rs index 021fd7c26a..aa1b9c69ab 100644 --- a/rust/src/engine-core-client/src/protocol/stats.rs +++ b/rust/src/engine-core-client/src/protocol/stats.rs @@ -278,9 +278,9 @@ pub use kv_connector::*; pub struct SchedulerStats { /// Number of requests in model execution batches. pub num_running_reqs: u64, - /// Length of the "waiting" request queue. + /// Number of waiting requests not deferred. pub num_waiting_reqs: u64, - /// Length of the "skipped waiting" queue. + /// Number of waiting requests deferred by transient constraints or a blocked status. #[serde(default)] pub num_skipped_waiting_reqs: u64, /// Internal DP load-balancing step counter. diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 505eef5919..91c93ed68f 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -2273,7 +2273,8 @@ def test_spec_decode_padding_skipped_with_prefill_in_batch(): def test_scheduler_stats_waiting_queues(): - """Test that scheduler stats correctly report waiting and skipped_waiting queues.""" + """Test that scheduler stats correctly report capacity-bound and blocked + waiting requests.""" # Create scheduler with limited capacity so we can have waiting requests scheduler = create_scheduler(max_num_batched_tokens=100) @@ -2286,18 +2287,18 @@ def test_scheduler_stats_waiting_queues(): for request in all_requests[:3]: scheduler.add_request(request) - # Manually add 2 more to skipped_waiting to simulate constraint-blocked + # Add 2 more that are blocked on grammar compilation for request in all_requests[3:]: - request.status = RequestStatus.WAITING_FOR_REMOTE_KVS - scheduler.skipped_waiting.add_request(request) + request.status = RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR + scheduler.add_request(request) # Schedule - this will schedule 2 requests, leaving 1 in waiting output = scheduler.schedule() # Verify: 2 scheduled, 1 still waiting on capacity, 2 blocked by constraints assert len(output.scheduled_new_reqs) == 2 - assert len(scheduler.waiting) == 1 - assert len(scheduler.skipped_waiting) == 2 + assert len(scheduler.waiting) == 3 + assert len(scheduler.deferred_waiting) == 2 # Call update_from_output() to get frontend-facing stat scheduled_req_ids = list(output.num_scheduled_tokens.keys()) @@ -2395,7 +2396,7 @@ def _step_until_done( def _num_waiting_requests(scheduler: Scheduler) -> int: - return len(scheduler.waiting) + len(scheduler.skipped_waiting) + return len(scheduler.waiting) + len(scheduler.kv_holding_waiting) def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]): @@ -2944,7 +2945,7 @@ def test_kv_connector_handles_preemption( if is_async: waiting_req_ids = [ req.request_id - for req in scheduler.skipped_waiting + for req in scheduler.kv_holding_waiting if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS ] assert len(waiting_req_ids) == 1 @@ -3807,8 +3808,8 @@ def test_schedule_skip_tokenizer_init_structured_output_request(): output = scheduler.schedule() assert len(output.scheduled_new_reqs) == 0 assert len(scheduler.running) == 0 - assert len(scheduler.waiting) == 0 - assert len(scheduler.skipped_waiting) == 1 + assert len(scheduler.waiting) == 1 + assert len(scheduler.deferred_waiting) == 1 @pytest.mark.parametrize("async_grammar", [True, False]) @@ -5227,7 +5228,7 @@ def test_prepend_skipped_requests_order(): req.status = RequestStatus.WAITING_FOR_REMOTE_KVS scheduler.waiting.remove_requests(expected_waiting_reqs[:2]) for req in expected_waiting_reqs[:2]: - scheduler.skipped_waiting.add_request(req) + scheduler.kv_holding_waiting.add_request(req) # schedule step # expect the first 2 waiting to be skipped, the third running, @@ -5237,21 +5238,18 @@ def test_prepend_skipped_requests_order(): # pop the third request which is expected to be running expected_waiting_reqs.pop(2) - # verify waiting order is preserved - waiting_reqs = list(scheduler.skipped_waiting) + list(scheduler.waiting) + # verify waiting order is preserved, with the KV-holding requests first + waiting_reqs = list(scheduler.kv_holding_waiting) + list(scheduler.waiting) assert waiting_reqs == expected_waiting_reqs -def test_remote_kv_promotion_keeps_fcfs_with_grammar_prefix(): +def test_remote_kv_promotion_drains_before_grammar_prefix(): scheduler = create_scheduler(max_num_seqs=1) scheduler.connector = Mock() scheduler.connector.get_num_new_matched_tokens.return_value = (0, False) requests = create_requests(num_requests=4) - for request in requests: - scheduler.add_request(request) - - req_grammar_1, req_grammar_2, req_remote, req_tail = list(scheduler.waiting) + req_grammar_1, req_grammar_2, req_remote, req_tail = requests # simulate two structured-output grammar requests at the waiting head # that become ready now. @@ -5260,31 +5258,28 @@ def test_remote_kv_promotion_keeps_fcfs_with_grammar_prefix(): req_grammar_2.status = RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR req_grammar_2.structured_output_request = Mock(grammar=object()) - # simulate a remote-KV request that is ready to be promoted now. + # simulate a remote-KV request that is ready to be promoted now. Parked + # loads hold KV blocks, so they drain ahead of the grammar requests. req_remote.status = RequestStatus.WAITING_FOR_REMOTE_KVS - scheduler.waiting.remove_requests([req_grammar_1, req_grammar_2, req_remote]) - scheduler.skipped_waiting.add_request(req_grammar_1) - scheduler.skipped_waiting.add_request(req_grammar_2) - scheduler.skipped_waiting.add_request(req_remote) + for request in requests: + scheduler.add_request(request) + assert list(scheduler.kv_holding_waiting) == [req_remote] scheduler.finished_recving_kv_req_ids.add(req_remote.request_id) scheduler._update_waiting_for_remote_kv = Mock() output = scheduler.schedule() assert output.scheduled_new_reqs - assert output.scheduled_new_reqs[0].req_id == req_grammar_1.request_id - waiting_req_ids = [ - req.request_id - for req in list(scheduler.skipped_waiting) + list(scheduler.waiting) - ] + assert output.scheduled_new_reqs[0].req_id == req_remote.request_id + waiting_req_ids = [req.request_id for req in scheduler.waiting] assert waiting_req_ids == [ + req_grammar_1.request_id, req_grammar_2.request_id, - req_remote.request_id, req_tail.request_id, ] -def test_fcfs_mixed_skipped_waiting_types_keep_order(): +def test_fcfs_mixed_blocked_types_remote_load_drains_first(): scheduler = create_scheduler(max_num_batched_tokens=20) scheduler._update_waiting_for_remote_kv = Mock() @@ -5305,7 +5300,8 @@ def test_fcfs_mixed_skipped_waiting_types_keep_order(): for req in (req_grammar, req_remote, req_stream, req_regular, req_tail): scheduler.add_request(req) scheduler.schedule() - assert list(scheduler.skipped_waiting) == [req_grammar, req_remote, req_stream] + assert list(scheduler.waiting) == [req_grammar, req_stream, req_tail] + assert list(scheduler.kv_holding_waiting) == [req_remote] scheduler.finish_requests(req_regular.request_id, RequestStatus.FINISHED_ABORTED) assert not scheduler.running @@ -5316,8 +5312,8 @@ def test_fcfs_mixed_skipped_waiting_types_keep_order(): second_output = scheduler.schedule() expected_order = [ - req_grammar.request_id, req_remote.request_id, + req_grammar.request_id, req_stream.request_id, req_tail.request_id, ] diff --git a/tests/v1/core/test_worker_slot_overflow.py b/tests/v1/core/test_worker_slot_overflow.py index 79366b0a23..fcf5cb8a9b 100644 --- a/tests/v1/core/test_worker_slot_overflow.py +++ b/tests/v1/core/test_worker_slot_overflow.py @@ -105,8 +105,8 @@ def test_streaming_pause_does_not_over_admit_worker_slot(): # still holds its slot. scheduler.update_from_output(out, _model_runner_output(["session"], [[STOP_TOKEN]])) assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ - # A blocked waiting status lands in skipped_waiting, not the main queue. - assert session in scheduler.skipped_waiting + # The paused session keeps its KV blocks, so it lands in kv_holding_waiting. + assert session in scheduler.kv_holding_waiting assert scheduler.num_waiting_for_streaming_input == 1 # A different request arrives while the session is paused. diff --git a/tests/v1/kv_connector/unit/test_error_propagation.py b/tests/v1/kv_connector/unit/test_error_propagation.py index 2f33ab18b9..91ec525f5c 100644 --- a/tests/v1/kv_connector/unit/test_error_propagation.py +++ b/tests/v1/kv_connector/unit/test_error_propagation.py @@ -119,7 +119,7 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler): scheduler_output = fail_scheduler.schedule() - assert len(fail_scheduler.skipped_waiting) == 1 + assert len(fail_scheduler.kv_holding_waiting) == 1 assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS assert request.num_computed_tokens == num_external_computed_tokens @@ -145,4 +145,5 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler): assert output.finish_reason == FinishReason.ERROR assert len(fail_scheduler.waiting) == 0 - assert len(fail_scheduler.skipped_waiting) == 0 + assert len(fail_scheduler.kv_holding_waiting) == 0 + assert not fail_scheduler.deferred_waiting diff --git a/tests/v1/kv_connector/unit/test_invalid_blocks_correctness.py b/tests/v1/kv_connector/unit/test_invalid_blocks_correctness.py index 41fce6e9ca..777bada58b 100644 --- a/tests/v1/kv_connector/unit/test_invalid_blocks_correctness.py +++ b/tests/v1/kv_connector/unit/test_invalid_blocks_correctness.py @@ -333,7 +333,7 @@ def test_async_recompute_blocks_not_cached_when_invalid( scheduler_output = recompute_scheduler.schedule() # request should be waiting for remote KVs - assert len(recompute_scheduler.skipped_waiting) == 1 + assert len(recompute_scheduler.kv_holding_waiting) == 1 assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS assert request.num_computed_tokens == num_external_computed_tokens diff --git a/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py b/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py index a8bc2d0e12..108eb049d9 100644 --- a/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py +++ b/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py @@ -111,8 +111,8 @@ def test_async_load_failure( scheduler_output = scheduler.schedule() assert len(scheduler.waiting) == 0 - assert len(scheduler.skipped_waiting) == 3 - for request in scheduler.skipped_waiting: + assert len(scheduler.kv_holding_waiting) == 3 + for request in scheduler.kv_holding_waiting: assert request.num_computed_tokens == num_external_computed_tokens assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS assert scheduler.connector.get_num_new_matched_tokens.call_count == 3 @@ -132,8 +132,8 @@ def test_async_load_failure( min_invalid_block_idx = min(invalid_block_idxs) assert len(scheduler.waiting) == 0 - assert len(scheduler.skipped_waiting) == 3 - for request in scheduler.skipped_waiting: + assert len(scheduler.kv_holding_waiting) == 3 + for request in scheduler.kv_holding_waiting: if request.request_id == request2.request_id: assert request.num_computed_tokens == ( min_invalid_block_idx * scheduler.block_size @@ -340,8 +340,9 @@ def test_async_progressive_load_failure( scheduler_output = scheduler.schedule() assert len(scheduler.waiting) == 0 - assert len(scheduler.skipped_waiting) == 1 - assert scheduler.skipped_waiting.peek_request().request_id == request.request_id + assert len(scheduler.kv_holding_waiting) == 1 + peeked = scheduler.kv_holding_waiting.peek_request() + assert peeked.request_id == request.request_id assert request.num_computed_tokens == num_external_computed_tokens assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS assert scheduler.connector.get_num_new_matched_tokens.call_count == 1 @@ -363,8 +364,9 @@ def test_async_progressive_load_failure( min_invalid_block_idx = min(min_invalid_block_idx, invalid_block_idx) assert len(scheduler.waiting) == 0 - assert len(scheduler.skipped_waiting) == 1 - assert scheduler.skipped_waiting.peek_request().request_id == request.request_id + assert len(scheduler.kv_holding_waiting) == 1 + peeked = scheduler.kv_holding_waiting.peek_request() + assert peeked.request_id == request.request_id assert request.num_computed_tokens == ( min_invalid_block_idx * scheduler.block_size ) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index d8b6fcb5a0..017dee10d8 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -3026,7 +3026,7 @@ def test_empty_recv_is_reported_only_when_awaited( request is named in neither _recving_transfers nor _failed_recv_reqs and never reaches finished_recving. The scheduler has no other way to release a WAITING_FOR_REMOTE_KVS request, and there is no timeout, so it sits in - skipped_waiting holding its blocks for the life of the process. + kv_holding_waiting holding its blocks for the life of the process. Notify-only (awaiting_kvs=False): request_finished seeding an empty recv to free P's blocks for a request aborted before it was scheduled, or a readback diff --git a/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py b/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py index caff9c9a49..daddd20b4a 100644 --- a/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py +++ b/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py @@ -25,7 +25,7 @@ pytestmark = pytest.mark.cpu_test def _num_waiting_requests(scheduler) -> int: - return len(scheduler.waiting) + len(scheduler.skipped_waiting) + return len(scheduler.waiting) + len(scheduler.kv_holding_waiting) def test_basic_lifecycle(): @@ -64,7 +64,7 @@ def test_basic_lifecycle(): # Req waiting for KVs with no computed/scheduled toks ... assert _num_waiting_requests(scheduler) == 1 - assert request in scheduler.skipped_waiting + assert request in scheduler.kv_holding_waiting assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS assert request.num_computed_tokens == NUM_TOKENS @@ -791,3 +791,71 @@ def test_async_load_reserves_blocks_for_promotion_margin(): scheduler_output = scheduler.schedule() assert req_a.status == RequestStatus.RUNNING assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0 + + +def test_deferred_lookup_does_not_block_parked_load(): + """A request with a deferred connector lookup must not strand an async + load queued behind it. + + req_a's lookup is deferred (connector returns None), so it is parked in + waiting holding no blocks; req_b is then admitted as an async + load holding most of the pool. Once req_a's lookup resolves it cannot + allocate, and the waiting scan stops at it every step. The parked load + must still be promotable: it lives in the KV-holding queue + that is drained first, so it runs, finishes, and frees its blocks for + req_a. + """ + vllm_config = create_vllm_config() + BLOCK_SIZE = vllm_config.cache_config.block_size + scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7 + + req_a = create_request( + request_id=1, block_size=BLOCK_SIZE, num_tokens=BLOCK_SIZE * 4, max_tokens=1 + ) + req_b = create_request( + request_id=2, + block_size=BLOCK_SIZE, + num_tokens=BLOCK_SIZE * 5, + do_remote_prefill=True, + max_tokens=1, + ) + scheduler.add_request(req_a) + scheduler.add_request(req_b) + + deferrals = [(None, False)] + + def lookup(request, num_computed_tokens): + if request is req_b: + return BLOCK_SIZE * 5, True + return deferrals.pop() if deferrals else (0, False) + + with patch.object( + scheduler.connector, "get_num_new_matched_tokens", side_effect=lookup + ): + # Step 1: req_a's lookup is deferred; req_b is admitted as an async + # load holding 5 of 7 usable blocks. + scheduler.schedule() + assert req_a.status == RequestStatus.WAITING + assert req_b.status == RequestStatus.WAITING_FOR_REMOTE_KVS + + # req_a's lookup resolves (no hit) while req_b's transfer is in + # flight; req_a needs 4 blocks but only 2 are free, so the waiting + # scan stops at it. req_b's transfer then finishes. + scheduler.update_from_output( + scheduler.schedule(), + create_model_runner_output([], finished_recving={req_b.request_id}), + ) + + # req_b must still be reached, promoted, and scheduled despite the + # scan stopping at req_a. + scheduler_output = scheduler.schedule() + assert req_b.status == RequestStatus.RUNNING + assert scheduler_output.num_scheduled_tokens[req_b.request_id] > 0 + + # req_b finishes and frees its blocks; now req_a fits and runs. + scheduler.update_from_output( + scheduler_output, create_model_runner_output([req_b]) + ) + scheduler_output = scheduler.schedule() + assert req_a.status == RequestStatus.RUNNING + assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0 diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index dda7c0f386..f154816369 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -54,6 +54,8 @@ def assert_scheduler_empty(scheduler: Scheduler): # Scheduler Metadata. assert len(scheduler.requests) == 0 assert len(scheduler.waiting) == 0 + assert len(scheduler.kv_holding_waiting) == 0 + assert not scheduler.deferred_waiting assert len(scheduler.running) == 0 assert len(scheduler.finished_req_ids) == 0 assert len(scheduler.finished_recving_kv_req_ids) == 0 diff --git a/tests/v1/streaming_input/test_scheduler_streaming.py b/tests/v1/streaming_input/test_scheduler_streaming.py index 45fe7e6c4b..8328298399 100644 --- a/tests/v1/streaming_input/test_scheduler_streaming.py +++ b/tests/v1/streaming_input/test_scheduler_streaming.py @@ -554,9 +554,9 @@ class TestStreamingScheduler(unittest.TestCase): eco_cycle2 = eco_dict_cycle2[session.client_index].outputs[0] assert eco_cycle2.finish_reason == FinishReason.STOP assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ - # Sessions paused for streaming input are blocked-waiting, so they - # live in the skipped_waiting queue rather than the main waiting queue. - assert session in scheduler.skipped_waiting + # Sessions paused for streaming input keep their KV blocks, so they + # live in the kv_holding_waiting queue. + assert session in scheduler.kv_holding_waiting assert session._all_token_ids == [1, 2, 3, 10, STOP_TOKEN] # CRITICAL ASSERTION: Cached prompt_token_ids STILL must not have changed diff --git a/vllm/v1/core/sched/request_queue.py b/vllm/v1/core/sched/request_queue.py index ce58e6e1c7..0a0277eb56 100644 --- a/vllm/v1/core/sched/request_queue.py +++ b/vllm/v1/core/sched/request_queue.py @@ -41,7 +41,7 @@ class RequestQueue(ABC): pass @abstractmethod - def prepend_requests(self, requests: "RequestQueue") -> None: + def prepend_requests(self, requests: Iterable[Request]) -> None: """Prepend all requests from another queue to the front of this queue.""" pass @@ -93,7 +93,7 @@ class FCFSRequestQueue(deque[Request], RequestQueue): """Prepend a request to the front of the queue.""" self.appendleft(request) - def prepend_requests(self, requests: RequestQueue) -> None: + def prepend_requests(self, requests: Iterable[Request]) -> None: """Prepend all requests from another queue to the front of this queue. @@ -163,7 +163,7 @@ class PriorityRequestQueue(RequestQueue): front. Requests are ordered by (priority, arrival_time).""" self.add_request(request) - def prepend_requests(self, requests: RequestQueue) -> None: + def prepend_requests(self, requests: Iterable[Request]) -> None: """Add all requests from another queue according to priority policy. Note: In a priority queue, there is no concept of prepending to the diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index f05b12575c..905415f357 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -210,8 +210,13 @@ class Scheduler(SchedulerInterface): ) from e # Priority queues for requests. self.waiting = create_request_queue(self.policy) - # requests skipped in waiting flow due async deps or constraints. - self.skipped_waiting = create_request_queue(self.policy) + # Waiting requests that hold KV blocks are always drained before + # self.waiting: a block-holder must never sit behind a request whose + # failed allocation would stop the scheduling scan. + self.kv_holding_waiting = create_request_queue(self.policy) + # Waiting requests that are deferred, i.e. were skipped by the scheduling + # scan or enqueued in a blocked status. + self.deferred_waiting: set[Request] = set() self.running: list[Request] = [] # The request IDs that are finished in between the previous and the @@ -606,9 +611,8 @@ class Scheduler(SchedulerInterface): # `long_prefill_token_threshold` exists to stop a long prefill from # starving other requests of the token budget. When it is the only # request there is nobody to starve, so let it use the whole budget. - num_eligible_reqs = ( - len(self.running) + len(self.waiting) + len(self.skipped_waiting) - ) + num_running, num_waiting = self.get_request_counts() + num_eligible_reqs = num_running + num_waiting long_prefill_token_threshold = ( self.scheduler_config.long_prefill_token_threshold if num_eligible_reqs > 1 @@ -867,10 +871,21 @@ class Scheduler(SchedulerInterface): # Next, schedule the WAITING requests. if not preempted_reqs and self._pause_state == PauseState.UNPAUSED: - step_skipped_waiting = create_request_queue(self.policy) + step_skipped_waiting: deque[Request] = deque() + step_skipped_kv_holding: deque[Request] = deque() - while (self.waiting or self.skipped_waiting) and token_budget > 0: - if input_budget <= draft_slots: + def skip_request(from_queue: RequestQueue) -> None: + request_to_skip = from_queue.pop_request() + self.deferred_waiting.add(request_to_skip) + if self._holds_kv_blocks(request_to_skip): + step_skipped_kv_holding.appendleft(request_to_skip) + else: + step_skipped_waiting.appendleft(request_to_skip) + + while token_budget > 0: + # Requests holding KV blocks are always drained first. + request_queue = self.kv_holding_waiting or self.waiting + if not request_queue or input_budget <= draft_slots: break # Paused streaming sessions (WAITING_FOR_STREAMING_REQ) are not # in `running` but still hold a model-runner request slot. @@ -878,38 +893,22 @@ class Scheduler(SchedulerInterface): if num_running >= self.max_num_active_reqs: break - request_queue = self._select_waiting_queue_for_scheduling() - assert request_queue is not None - request = request_queue.peek_request() request_id = request.request_id - # try to promote blocked statuses while traversing skipped queue. - if self._is_blocked_waiting_status( - request.status - ) and not self._try_promote_blocked_waiting_request(request): - if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS: - logger.debug( - "%s is still in WAITING_FOR_REMOTE_KVS state.", - request_id, - ) - request_queue.pop_request() - step_skipped_waiting.prepend_request(request) + ready_to_schedule = self._handle_blocked_waiting_request(request) + if not ready_to_schedule: + skip_request(request_queue) continue - if ( - request.num_stale_output_tokens > 0 - and not request.drop_stale_output - ): + if request.num_stale_output_tokens and not request.drop_stale_output: # Deliverable stale output still in flight: resuming now # could resample a position that output later delivers. # It drains within the pipeline depth. - request_queue.pop_request() - step_skipped_waiting.prepend_request(request) + skip_request(request_queue) continue - # Check that adding the request still respects the max_loras - # constraint. + # Check that adding the request still respects the max_loras constraint. if ( self.lora_config and request.lora_request @@ -919,8 +918,7 @@ class Scheduler(SchedulerInterface): ) ): # Scheduling would exceed max_loras, skip. - request_queue.pop_request() - step_skipped_waiting.prepend_request(request) + skip_request(request_queue) continue num_external_computed_tokens = 0 @@ -959,8 +957,7 @@ class Scheduler(SchedulerInterface): # The request cannot be scheduled because # the KVConnector couldn't determine # the number of matched tokens. - request_queue.pop_request() - step_skipped_waiting.prepend_request(request) + skip_request(request_queue) continue if self.prefix_replay_tokens: @@ -1030,8 +1027,7 @@ class Scheduler(SchedulerInterface): # Skip request with pending mm encoding prefetches if self._ec_transfer_pending(request, num_computed_tokens): - request_queue.pop_request() - step_skipped_waiting.prepend_request(request) + skip_request(request_queue) continue # Track first scheduled prefill, not post-preemption repeat prefills @@ -1052,8 +1048,7 @@ class Scheduler(SchedulerInterface): num_computed_tokens = request.num_computed_tokens if self._ec_transfer_pending(request, num_computed_tokens): - request_queue.pop_request() - step_skipped_waiting.prepend_request(request) + skip_request(request_queue) continue encoder_inputs_to_schedule = None @@ -1260,12 +1255,10 @@ class Scheduler(SchedulerInterface): request, num_new_local_computed_tokens ) - request = request_queue.pop_request() if load_kv_async: # If loading async, allocate memory and put request # into the WAITING_FOR_REMOTE_KV state. request.status = RequestStatus.WAITING_FOR_REMOTE_KVS - step_skipped_waiting.prepend_request(request) # Set num_computed_tokens even though KVs are not yet loaded. # request.num_computed_tokens will not be used anywhere until # the request finished the KV transfer. @@ -1286,13 +1279,16 @@ class Scheduler(SchedulerInterface): # overwrite; the zeroing could race the write. self._skip_zero_block_ids.update( self.kv_cache_manager.get_zeroing_block_ids_in_range( - request.request_id, + request_id, num_new_local_computed_tokens, num_computed_tokens, ) ) + skip_request(request_queue) continue + request = request_queue.pop_request() + self.deferred_waiting.discard(request) self.running.append(request) if num_external_computed_tokens > 0: # load_kv_async is False here @@ -1343,13 +1339,16 @@ class Scheduler(SchedulerInterface): self.ec_connector.update_state_after_alloc(request, i) # re-queue requests skipped in this pass ahead of older skipped items. + if step_skipped_kv_holding: + self.kv_holding_waiting.prepend_requests(step_skipped_kv_holding) if step_skipped_waiting: - self.skipped_waiting.prepend_requests(step_skipped_waiting) + self.waiting.prepend_requests(step_skipped_waiting) # DP prefill balancing: on a step that admitted prefills (release), # record whether it was capacity-bound. if not defer_prefills: - self.prefill_capacity_bound = bool(self.waiting) + _, num_waiting = self.get_request_counts() + self.prefill_capacity_bound = num_waiting > len(self.deferred_waiting) # Check if the scheduling constraints are satisfied. total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) @@ -2237,7 +2236,8 @@ class Scheduler(SchedulerInterface): if stopped_preempted_reqs: # This is a rare case and unlikely to impact performance. self.waiting.remove_requests(stopped_preempted_reqs) - self.skipped_waiting.remove_requests(stopped_preempted_reqs) + self.kv_holding_waiting.remove_requests(stopped_preempted_reqs) + self.deferred_waiting.difference_update(stopped_preempted_reqs) error_req_ids = set(self.grammar_compile_error_reqs) self.grammar_compile_error_reqs.clear() @@ -2381,24 +2381,27 @@ class Scheduler(SchedulerInterface): RequestStatus.WAITING_FOR_STREAMING_REQ, ) + @staticmethod + def _holds_kv_blocks(request: Request) -> bool: + # Whether a request currently has kv blocks allocated. + # While always the case when status is WAITING_FOR_REMOTE_KVS + # or WAITING_FOR_STREAMING_REQ, there are such cases where it's + # WAITING or PREEMPTED. + return ( + request.num_computed_tokens > 0 + # num_computed_tokens is reset to 0 if kv transfer fails for requests in + # WAITING_FOR_REMOTE_KVS state, before their allocated blocks are freed. + or request.status == RequestStatus.WAITING_FOR_REMOTE_KVS + ) + def _enqueue_waiting_request(self, request: Request) -> None: if self._is_blocked_waiting_status(request.status): - self.skipped_waiting.add_request(request) + self.deferred_waiting.add(request) + if self._holds_kv_blocks(request): + self.kv_holding_waiting.add_request(request) else: self.waiting.add_request(request) - def _select_waiting_queue_for_scheduling(self) -> RequestQueue | None: - if self.policy == SchedulingPolicy.FCFS: - return self.skipped_waiting or self.waiting or None - - # PRIORITY mode: compare queue heads when both queues are non-empty. - if self.waiting and self.skipped_waiting: - waiting_req = self.waiting.peek_request() - skipped_req = self.skipped_waiting.peek_request() - return self.waiting if waiting_req < skipped_req else self.skipped_waiting - - return self.waiting or self.skipped_waiting or None - def _handle_stopped_request(self, request: Request) -> bool: """Return True if finished (can be False for resumable requests).""" if not request.resumable: @@ -2544,7 +2547,7 @@ class Scheduler(SchedulerInterface): def get_request_counts(self) -> tuple[int, int]: """Returns (num_running_reqs, num_waiting_reqs).""" - return len(self.running), len(self.waiting) + len(self.skipped_waiting) + return len(self.running), len(self.waiting) + len(self.kv_holding_waiting) def get_kv_cache_usage(self) -> float: """Returns the fraction of the KV cache currently in use (0.0-1.0).""" @@ -2625,7 +2628,8 @@ class Scheduler(SchedulerInterface): self.running = remove_all(self.running, running_requests_to_remove) if waiting_requests_to_remove: self.waiting.remove_requests(waiting_requests_to_remove) - self.skipped_waiting.remove_requests(waiting_requests_to_remove) + self.kv_holding_waiting.remove_requests(waiting_requests_to_remove) + self.deferred_waiting.difference_update(waiting_requests_to_remove) # Second pass: set status and free requests for request in valid_requests: @@ -2733,14 +2737,11 @@ class Scheduler(SchedulerInterface): def get_num_unfinished_requests(self) -> int: if self._pause_state == PauseState.PAUSED_ALL: return 0 + num_running, num_waiting = self.get_request_counts() if self._pause_state == PauseState.PAUSED_NEW: - return len(self.running) - num_waiting = ( - len(self.waiting) - + len(self.skipped_waiting) - - self.num_waiting_for_streaming_input - ) - return num_waiting + len(self.running) + return num_running + num_waiting -= self.num_waiting_for_streaming_input + return num_waiting + num_running def has_finished_requests(self) -> bool: if self.finished_req_ids: @@ -2749,10 +2750,8 @@ class Scheduler(SchedulerInterface): return False # Finished requests waiting on delayed connector cleanup remain in # self.requests after they have been removed from scheduling queues. - num_in_queues = ( - len(self.waiting) + len(self.skipped_waiting) + len(self.running) - ) - return len(self.requests) > num_in_queues + num_running, num_waiting = self.get_request_counts() + return len(self.requests) > (num_running + num_waiting) def has_requests(self) -> bool: # Override the interface default to also keep the engine alive while a @@ -2884,10 +2883,12 @@ class Scheduler(SchedulerInterface): ec_connector_stats_payload = ( ec_connector_stats.data if ec_connector_stats else None ) + num_running, num_waiting = self.get_request_counts() + num_deferred = len(self.deferred_waiting) return SchedulerStats( - num_running_reqs=len(self.running), - num_waiting_reqs=len(self.waiting), - num_skipped_waiting_reqs=len(self.skipped_waiting), + num_running_reqs=num_running, + num_waiting_reqs=num_waiting - num_deferred, + num_skipped_waiting_reqs=num_deferred, kv_cache_usage=self.kv_cache_manager.usage, prefix_cache_stats=prefix_cache_stats, connector_prefix_cache_stats=connector_prefix_cache_stats, @@ -3093,39 +3094,39 @@ class Scheduler(SchedulerInterface): request.num_computed_tokens = request.num_tokens - 1 self.finished_recving_kv_req_ids.remove(request.request_id) - def _try_promote_blocked_waiting_request(self, request: Request) -> bool: - """Try to promote a blocked waiting request back to schedulable states.""" + def _handle_blocked_waiting_request(self, request: Request) -> bool: + """Returns True if scheduling of this request should proceed, False if + it should be skipped for this step.""" if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS: - # finished_recving_kv_req_ids is populated during - # update_from_output(), based on worker-side connector signals - # in KVConnectorOutput.finished_recving - if request.request_id not in self.finished_recving_kv_req_ids: + # Check whether async loading requests are finished. + request_id = request.request_id + if request_id not in self.finished_recving_kv_req_ids: + logger.debug("%s is still in WAITING_FOR_REMOTE_KVS state.", request_id) return False + # Promote request to waiting or preempted. self._update_waiting_for_remote_kv(request) if request.num_preemptions: request.status = RequestStatus.PREEMPTED else: request.status = RequestStatus.WAITING - return True - if request.status == RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR: - structured_output_req = request.structured_output_request - if not structured_output_req or structured_output_req.grammar is None: + elif request.status == RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR: + # Check whether grammar compilation has finished. + struct_output_req = request.structured_output_request + if not struct_output_req or struct_output_req.grammar is None: return False - if isinstance(structured_output_req.grammar, Exception): + if isinstance(struct_output_req.grammar, Exception): self.grammar_compile_error_reqs.add(request.request_id) return False + # Grammar is ready, promote to waiting state. request.status = RequestStatus.WAITING - return True - if request.status == RequestStatus.WAITING_FOR_STREAMING_REQ: + elif request.status == RequestStatus.WAITING_FOR_STREAMING_REQ: + # Streaming input request waiting for more input. assert not request.streaming_queue return False - raise AssertionError( - "Unexpected blocked waiting status in promotion: " - f"{request.status.name} for request {request.request_id}" - ) + return True def _update_from_kv_xfer_finished(self, kv_connector_output: KVConnectorOutput): """KV Connector: update the scheduler state based on the output. @@ -3298,7 +3299,7 @@ class Scheduler(SchedulerInterface): # handle async KV loads (not cached yet, evict_blocks=False) async_load_reqs = ( req - for req in self.skipped_waiting + for req in self.kv_holding_waiting if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS ) async_failed_req_ids, num_failed_tokens, _ = ( diff --git a/vllm/v1/metrics/stats.py b/vllm/v1/metrics/stats.py index 66088c7954..da88c9379a 100644 --- a/vllm/v1/metrics/stats.py +++ b/vllm/v1/metrics/stats.py @@ -188,8 +188,9 @@ class SchedulerStats: num_running_reqs: int = 0 - num_waiting_reqs: int = 0 # length of the "waiting" request queue - num_skipped_waiting_reqs: int = 0 # length of the "skipped waiting" queue + num_waiting_reqs: int = 0 # waiting requests not deferred + # waiting requests deferred by transient constraints or a blocked status + num_skipped_waiting_reqs: int = 0 # These are used for internal DP load-balancing. step_counter: int = 0