[Core] Rework scheduler skipped_waiting queue (#58947)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-09-28 20:03:16 +00:00
committed by GitHub
co-authored by Robert Shaw
parent 2619b6b72b
commit 42dbe4578c
13 changed files with 220 additions and 149 deletions
@@ -278,9 +278,9 @@ pub use kv_connector::*;
pub struct SchedulerStats {
/// Number of requests in model execution batches.
pub num_running_reqs: u64,
/// Length of the "waiting" request queue.
/// Number of waiting requests not deferred.
pub num_waiting_reqs: u64,
/// Length of the "skipped waiting" queue.
/// Number of waiting requests deferred by transient constraints or a blocked status.
#[serde(default)]
pub num_skipped_waiting_reqs: u64,
/// Internal DP load-balancing step counter.
+28 -32
View File
@@ -2273,7 +2273,8 @@ def test_spec_decode_padding_skipped_with_prefill_in_batch():
def test_scheduler_stats_waiting_queues():
"""Test that scheduler stats correctly report waiting and skipped_waiting queues."""
"""Test that scheduler stats correctly report capacity-bound and blocked
waiting requests."""
# Create scheduler with limited capacity so we can have waiting requests
scheduler = create_scheduler(max_num_batched_tokens=100)
@@ -2286,18 +2287,18 @@ def test_scheduler_stats_waiting_queues():
for request in all_requests[:3]:
scheduler.add_request(request)
# Manually add 2 more to skipped_waiting to simulate constraint-blocked
# Add 2 more that are blocked on grammar compilation
for request in all_requests[3:]:
request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
scheduler.skipped_waiting.add_request(request)
request.status = RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR
scheduler.add_request(request)
# Schedule - this will schedule 2 requests, leaving 1 in waiting
output = scheduler.schedule()
# Verify: 2 scheduled, 1 still waiting on capacity, 2 blocked by constraints
assert len(output.scheduled_new_reqs) == 2
assert len(scheduler.waiting) == 1
assert len(scheduler.skipped_waiting) == 2
assert len(scheduler.waiting) == 3
assert len(scheduler.deferred_waiting) == 2
# Call update_from_output() to get frontend-facing stat
scheduled_req_ids = list(output.num_scheduled_tokens.keys())
@@ -2395,7 +2396,7 @@ def _step_until_done(
def _num_waiting_requests(scheduler: Scheduler) -> int:
return len(scheduler.waiting) + len(scheduler.skipped_waiting)
return len(scheduler.waiting) + len(scheduler.kv_holding_waiting)
def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]):
@@ -2944,7 +2945,7 @@ def test_kv_connector_handles_preemption(
if is_async:
waiting_req_ids = [
req.request_id
for req in scheduler.skipped_waiting
for req in scheduler.kv_holding_waiting
if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
]
assert len(waiting_req_ids) == 1
@@ -3807,8 +3808,8 @@ def test_schedule_skip_tokenizer_init_structured_output_request():
output = scheduler.schedule()
assert len(output.scheduled_new_reqs) == 0
assert len(scheduler.running) == 0
assert len(scheduler.waiting) == 0
assert len(scheduler.skipped_waiting) == 1
assert len(scheduler.waiting) == 1
assert len(scheduler.deferred_waiting) == 1
@pytest.mark.parametrize("async_grammar", [True, False])
@@ -5227,7 +5228,7 @@ def test_prepend_skipped_requests_order():
req.status = RequestStatus.WAITING_FOR_REMOTE_KVS
scheduler.waiting.remove_requests(expected_waiting_reqs[:2])
for req in expected_waiting_reqs[:2]:
scheduler.skipped_waiting.add_request(req)
scheduler.kv_holding_waiting.add_request(req)
# schedule step
# expect the first 2 waiting to be skipped, the third running,
@@ -5237,21 +5238,18 @@ def test_prepend_skipped_requests_order():
# pop the third request which is expected to be running
expected_waiting_reqs.pop(2)
# verify waiting order is preserved
waiting_reqs = list(scheduler.skipped_waiting) + list(scheduler.waiting)
# verify waiting order is preserved, with the KV-holding requests first
waiting_reqs = list(scheduler.kv_holding_waiting) + list(scheduler.waiting)
assert waiting_reqs == expected_waiting_reqs
def test_remote_kv_promotion_keeps_fcfs_with_grammar_prefix():
def test_remote_kv_promotion_drains_before_grammar_prefix():
scheduler = create_scheduler(max_num_seqs=1)
scheduler.connector = Mock()
scheduler.connector.get_num_new_matched_tokens.return_value = (0, False)
requests = create_requests(num_requests=4)
for request in requests:
scheduler.add_request(request)
req_grammar_1, req_grammar_2, req_remote, req_tail = list(scheduler.waiting)
req_grammar_1, req_grammar_2, req_remote, req_tail = requests
# simulate two structured-output grammar requests at the waiting head
# that become ready now.
@@ -5260,31 +5258,28 @@ def test_remote_kv_promotion_keeps_fcfs_with_grammar_prefix():
req_grammar_2.status = RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR
req_grammar_2.structured_output_request = Mock(grammar=object())
# simulate a remote-KV request that is ready to be promoted now.
# simulate a remote-KV request that is ready to be promoted now. Parked
# loads hold KV blocks, so they drain ahead of the grammar requests.
req_remote.status = RequestStatus.WAITING_FOR_REMOTE_KVS
scheduler.waiting.remove_requests([req_grammar_1, req_grammar_2, req_remote])
scheduler.skipped_waiting.add_request(req_grammar_1)
scheduler.skipped_waiting.add_request(req_grammar_2)
scheduler.skipped_waiting.add_request(req_remote)
for request in requests:
scheduler.add_request(request)
assert list(scheduler.kv_holding_waiting) == [req_remote]
scheduler.finished_recving_kv_req_ids.add(req_remote.request_id)
scheduler._update_waiting_for_remote_kv = Mock()
output = scheduler.schedule()
assert output.scheduled_new_reqs
assert output.scheduled_new_reqs[0].req_id == req_grammar_1.request_id
waiting_req_ids = [
req.request_id
for req in list(scheduler.skipped_waiting) + list(scheduler.waiting)
]
assert output.scheduled_new_reqs[0].req_id == req_remote.request_id
waiting_req_ids = [req.request_id for req in scheduler.waiting]
assert waiting_req_ids == [
req_grammar_1.request_id,
req_grammar_2.request_id,
req_remote.request_id,
req_tail.request_id,
]
def test_fcfs_mixed_skipped_waiting_types_keep_order():
def test_fcfs_mixed_blocked_types_remote_load_drains_first():
scheduler = create_scheduler(max_num_batched_tokens=20)
scheduler._update_waiting_for_remote_kv = Mock()
@@ -5305,7 +5300,8 @@ def test_fcfs_mixed_skipped_waiting_types_keep_order():
for req in (req_grammar, req_remote, req_stream, req_regular, req_tail):
scheduler.add_request(req)
scheduler.schedule()
assert list(scheduler.skipped_waiting) == [req_grammar, req_remote, req_stream]
assert list(scheduler.waiting) == [req_grammar, req_stream, req_tail]
assert list(scheduler.kv_holding_waiting) == [req_remote]
scheduler.finish_requests(req_regular.request_id, RequestStatus.FINISHED_ABORTED)
assert not scheduler.running
@@ -5316,8 +5312,8 @@ def test_fcfs_mixed_skipped_waiting_types_keep_order():
second_output = scheduler.schedule()
expected_order = [
req_grammar.request_id,
req_remote.request_id,
req_grammar.request_id,
req_stream.request_id,
req_tail.request_id,
]
+2 -2
View File
@@ -105,8 +105,8 @@ def test_streaming_pause_does_not_over_admit_worker_slot():
# still holds its slot.
scheduler.update_from_output(out, _model_runner_output(["session"], [[STOP_TOKEN]]))
assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ
# A blocked waiting status lands in skipped_waiting, not the main queue.
assert session in scheduler.skipped_waiting
# The paused session keeps its KV blocks, so it lands in kv_holding_waiting.
assert session in scheduler.kv_holding_waiting
assert scheduler.num_waiting_for_streaming_input == 1
# A different request arrives while the session is paused.
@@ -119,7 +119,7 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler):
scheduler_output = fail_scheduler.schedule()
assert len(fail_scheduler.skipped_waiting) == 1
assert len(fail_scheduler.kv_holding_waiting) == 1
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert request.num_computed_tokens == num_external_computed_tokens
@@ -145,4 +145,5 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler):
assert output.finish_reason == FinishReason.ERROR
assert len(fail_scheduler.waiting) == 0
assert len(fail_scheduler.skipped_waiting) == 0
assert len(fail_scheduler.kv_holding_waiting) == 0
assert not fail_scheduler.deferred_waiting
@@ -333,7 +333,7 @@ def test_async_recompute_blocks_not_cached_when_invalid(
scheduler_output = recompute_scheduler.schedule()
# request should be waiting for remote KVs
assert len(recompute_scheduler.skipped_waiting) == 1
assert len(recompute_scheduler.kv_holding_waiting) == 1
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert request.num_computed_tokens == num_external_computed_tokens
@@ -111,8 +111,8 @@ def test_async_load_failure(
scheduler_output = scheduler.schedule()
assert len(scheduler.waiting) == 0
assert len(scheduler.skipped_waiting) == 3
for request in scheduler.skipped_waiting:
assert len(scheduler.kv_holding_waiting) == 3
for request in scheduler.kv_holding_waiting:
assert request.num_computed_tokens == num_external_computed_tokens
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert scheduler.connector.get_num_new_matched_tokens.call_count == 3
@@ -132,8 +132,8 @@ def test_async_load_failure(
min_invalid_block_idx = min(invalid_block_idxs)
assert len(scheduler.waiting) == 0
assert len(scheduler.skipped_waiting) == 3
for request in scheduler.skipped_waiting:
assert len(scheduler.kv_holding_waiting) == 3
for request in scheduler.kv_holding_waiting:
if request.request_id == request2.request_id:
assert request.num_computed_tokens == (
min_invalid_block_idx * scheduler.block_size
@@ -340,8 +340,9 @@ def test_async_progressive_load_failure(
scheduler_output = scheduler.schedule()
assert len(scheduler.waiting) == 0
assert len(scheduler.skipped_waiting) == 1
assert scheduler.skipped_waiting.peek_request().request_id == request.request_id
assert len(scheduler.kv_holding_waiting) == 1
peeked = scheduler.kv_holding_waiting.peek_request()
assert peeked.request_id == request.request_id
assert request.num_computed_tokens == num_external_computed_tokens
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert scheduler.connector.get_num_new_matched_tokens.call_count == 1
@@ -363,8 +364,9 @@ def test_async_progressive_load_failure(
min_invalid_block_idx = min(min_invalid_block_idx, invalid_block_idx)
assert len(scheduler.waiting) == 0
assert len(scheduler.skipped_waiting) == 1
assert scheduler.skipped_waiting.peek_request().request_id == request.request_id
assert len(scheduler.kv_holding_waiting) == 1
peeked = scheduler.kv_holding_waiting.peek_request()
assert peeked.request_id == request.request_id
assert request.num_computed_tokens == (
min_invalid_block_idx * scheduler.block_size
)
@@ -3026,7 +3026,7 @@ def test_empty_recv_is_reported_only_when_awaited(
request is named in neither _recving_transfers nor _failed_recv_reqs and
never reaches finished_recving. The scheduler has no other way to release a
WAITING_FOR_REMOTE_KVS request, and there is no timeout, so it sits in
skipped_waiting holding its blocks for the life of the process.
kv_holding_waiting holding its blocks for the life of the process.
Notify-only (awaiting_kvs=False): request_finished seeding an empty recv to
free P's blocks for a request aborted before it was scheduled, or a readback
@@ -25,7 +25,7 @@ pytestmark = pytest.mark.cpu_test
def _num_waiting_requests(scheduler) -> int:
return len(scheduler.waiting) + len(scheduler.skipped_waiting)
return len(scheduler.waiting) + len(scheduler.kv_holding_waiting)
def test_basic_lifecycle():
@@ -64,7 +64,7 @@ def test_basic_lifecycle():
# Req waiting for KVs with no computed/scheduled toks ...
assert _num_waiting_requests(scheduler) == 1
assert request in scheduler.skipped_waiting
assert request in scheduler.kv_holding_waiting
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert request.num_computed_tokens == NUM_TOKENS
@@ -791,3 +791,71 @@ def test_async_load_reserves_blocks_for_promotion_margin():
scheduler_output = scheduler.schedule()
assert req_a.status == RequestStatus.RUNNING
assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0
def test_deferred_lookup_does_not_block_parked_load():
"""A request with a deferred connector lookup must not strand an async
load queued behind it.
req_a's lookup is deferred (connector returns None), so it is parked in
waiting holding no blocks; req_b is then admitted as an async
load holding most of the pool. Once req_a's lookup resolves it cannot
allocate, and the waiting scan stops at it every step. The parked load
must still be promotable: it lives in the KV-holding queue
that is drained first, so it runs, finishes, and frees its blocks for
req_a.
"""
vllm_config = create_vllm_config()
BLOCK_SIZE = vllm_config.cache_config.block_size
scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7
req_a = create_request(
request_id=1, block_size=BLOCK_SIZE, num_tokens=BLOCK_SIZE * 4, max_tokens=1
)
req_b = create_request(
request_id=2,
block_size=BLOCK_SIZE,
num_tokens=BLOCK_SIZE * 5,
do_remote_prefill=True,
max_tokens=1,
)
scheduler.add_request(req_a)
scheduler.add_request(req_b)
deferrals = [(None, False)]
def lookup(request, num_computed_tokens):
if request is req_b:
return BLOCK_SIZE * 5, True
return deferrals.pop() if deferrals else (0, False)
with patch.object(
scheduler.connector, "get_num_new_matched_tokens", side_effect=lookup
):
# Step 1: req_a's lookup is deferred; req_b is admitted as an async
# load holding 5 of 7 usable blocks.
scheduler.schedule()
assert req_a.status == RequestStatus.WAITING
assert req_b.status == RequestStatus.WAITING_FOR_REMOTE_KVS
# req_a's lookup resolves (no hit) while req_b's transfer is in
# flight; req_a needs 4 blocks but only 2 are free, so the waiting
# scan stops at it. req_b's transfer then finishes.
scheduler.update_from_output(
scheduler.schedule(),
create_model_runner_output([], finished_recving={req_b.request_id}),
)
# req_b must still be reached, promoted, and scheduled despite the
# scan stopping at req_a.
scheduler_output = scheduler.schedule()
assert req_b.status == RequestStatus.RUNNING
assert scheduler_output.num_scheduled_tokens[req_b.request_id] > 0
# req_b finishes and frees its blocks; now req_a fits and runs.
scheduler.update_from_output(
scheduler_output, create_model_runner_output([req_b])
)
scheduler_output = scheduler.schedule()
assert req_a.status == RequestStatus.RUNNING
assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0
+2
View File
@@ -54,6 +54,8 @@ def assert_scheduler_empty(scheduler: Scheduler):
# Scheduler Metadata.
assert len(scheduler.requests) == 0
assert len(scheduler.waiting) == 0
assert len(scheduler.kv_holding_waiting) == 0
assert not scheduler.deferred_waiting
assert len(scheduler.running) == 0
assert len(scheduler.finished_req_ids) == 0
assert len(scheduler.finished_recving_kv_req_ids) == 0
@@ -554,9 +554,9 @@ class TestStreamingScheduler(unittest.TestCase):
eco_cycle2 = eco_dict_cycle2[session.client_index].outputs[0]
assert eco_cycle2.finish_reason == FinishReason.STOP
assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ
# Sessions paused for streaming input are blocked-waiting, so they
# live in the skipped_waiting queue rather than the main waiting queue.
assert session in scheduler.skipped_waiting
# Sessions paused for streaming input keep their KV blocks, so they
# live in the kv_holding_waiting queue.
assert session in scheduler.kv_holding_waiting
assert session._all_token_ids == [1, 2, 3, 10, STOP_TOKEN]
# CRITICAL ASSERTION: Cached prompt_token_ids STILL must not have changed
+3 -3
View File
@@ -41,7 +41,7 @@ class RequestQueue(ABC):
pass
@abstractmethod
def prepend_requests(self, requests: "RequestQueue") -> None:
def prepend_requests(self, requests: Iterable[Request]) -> None:
"""Prepend all requests from another queue to the front of this
queue."""
pass
@@ -93,7 +93,7 @@ class FCFSRequestQueue(deque[Request], RequestQueue):
"""Prepend a request to the front of the queue."""
self.appendleft(request)
def prepend_requests(self, requests: RequestQueue) -> None:
def prepend_requests(self, requests: Iterable[Request]) -> None:
"""Prepend all requests from another queue to the front of this
queue.
@@ -163,7 +163,7 @@ class PriorityRequestQueue(RequestQueue):
front. Requests are ordered by (priority, arrival_time)."""
self.add_request(request)
def prepend_requests(self, requests: RequestQueue) -> None:
def prepend_requests(self, requests: Iterable[Request]) -> None:
"""Add all requests from another queue according to priority policy.
Note: In a priority queue, there is no concept of prepending to the
+92 -91
View File
@@ -210,8 +210,13 @@ class Scheduler(SchedulerInterface):
) from e
# Priority queues for requests.
self.waiting = create_request_queue(self.policy)
# requests skipped in waiting flow due async deps or constraints.
self.skipped_waiting = create_request_queue(self.policy)
# Waiting requests that hold KV blocks are always drained before
# self.waiting: a block-holder must never sit behind a request whose
# failed allocation would stop the scheduling scan.
self.kv_holding_waiting = create_request_queue(self.policy)
# Waiting requests that are deferred, i.e. were skipped by the scheduling
# scan or enqueued in a blocked status.
self.deferred_waiting: set[Request] = set()
self.running: list[Request] = []
# The request IDs that are finished in between the previous and the
@@ -606,9 +611,8 @@ class Scheduler(SchedulerInterface):
# `long_prefill_token_threshold` exists to stop a long prefill from
# starving other requests of the token budget. When it is the only
# request there is nobody to starve, so let it use the whole budget.
num_eligible_reqs = (
len(self.running) + len(self.waiting) + len(self.skipped_waiting)
)
num_running, num_waiting = self.get_request_counts()
num_eligible_reqs = num_running + num_waiting
long_prefill_token_threshold = (
self.scheduler_config.long_prefill_token_threshold
if num_eligible_reqs > 1
@@ -867,10 +871,21 @@ class Scheduler(SchedulerInterface):
# Next, schedule the WAITING requests.
if not preempted_reqs and self._pause_state == PauseState.UNPAUSED:
step_skipped_waiting = create_request_queue(self.policy)
step_skipped_waiting: deque[Request] = deque()
step_skipped_kv_holding: deque[Request] = deque()
while (self.waiting or self.skipped_waiting) and token_budget > 0:
if input_budget <= draft_slots:
def skip_request(from_queue: RequestQueue) -> None:
request_to_skip = from_queue.pop_request()
self.deferred_waiting.add(request_to_skip)
if self._holds_kv_blocks(request_to_skip):
step_skipped_kv_holding.appendleft(request_to_skip)
else:
step_skipped_waiting.appendleft(request_to_skip)
while token_budget > 0:
# Requests holding KV blocks are always drained first.
request_queue = self.kv_holding_waiting or self.waiting
if not request_queue or input_budget <= draft_slots:
break
# Paused streaming sessions (WAITING_FOR_STREAMING_REQ) are not
# in `running` but still hold a model-runner request slot.
@@ -878,38 +893,22 @@ class Scheduler(SchedulerInterface):
if num_running >= self.max_num_active_reqs:
break
request_queue = self._select_waiting_queue_for_scheduling()
assert request_queue is not None
request = request_queue.peek_request()
request_id = request.request_id
# try to promote blocked statuses while traversing skipped queue.
if self._is_blocked_waiting_status(
request.status
) and not self._try_promote_blocked_waiting_request(request):
if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS:
logger.debug(
"%s is still in WAITING_FOR_REMOTE_KVS state.",
request_id,
)
request_queue.pop_request()
step_skipped_waiting.prepend_request(request)
ready_to_schedule = self._handle_blocked_waiting_request(request)
if not ready_to_schedule:
skip_request(request_queue)
continue
if (
request.num_stale_output_tokens > 0
and not request.drop_stale_output
):
if request.num_stale_output_tokens and not request.drop_stale_output:
# Deliverable stale output still in flight: resuming now
# could resample a position that output later delivers.
# It drains within the pipeline depth.
request_queue.pop_request()
step_skipped_waiting.prepend_request(request)
skip_request(request_queue)
continue
# Check that adding the request still respects the max_loras
# constraint.
# Check that adding the request still respects the max_loras constraint.
if (
self.lora_config
and request.lora_request
@@ -919,8 +918,7 @@ class Scheduler(SchedulerInterface):
)
):
# Scheduling would exceed max_loras, skip.
request_queue.pop_request()
step_skipped_waiting.prepend_request(request)
skip_request(request_queue)
continue
num_external_computed_tokens = 0
@@ -959,8 +957,7 @@ class Scheduler(SchedulerInterface):
# The request cannot be scheduled because
# the KVConnector couldn't determine
# the number of matched tokens.
request_queue.pop_request()
step_skipped_waiting.prepend_request(request)
skip_request(request_queue)
continue
if self.prefix_replay_tokens:
@@ -1030,8 +1027,7 @@ class Scheduler(SchedulerInterface):
# Skip request with pending mm encoding prefetches
if self._ec_transfer_pending(request, num_computed_tokens):
request_queue.pop_request()
step_skipped_waiting.prepend_request(request)
skip_request(request_queue)
continue
# Track first scheduled prefill, not post-preemption repeat prefills
@@ -1052,8 +1048,7 @@ class Scheduler(SchedulerInterface):
num_computed_tokens = request.num_computed_tokens
if self._ec_transfer_pending(request, num_computed_tokens):
request_queue.pop_request()
step_skipped_waiting.prepend_request(request)
skip_request(request_queue)
continue
encoder_inputs_to_schedule = None
@@ -1260,12 +1255,10 @@ class Scheduler(SchedulerInterface):
request, num_new_local_computed_tokens
)
request = request_queue.pop_request()
if load_kv_async:
# If loading async, allocate memory and put request
# into the WAITING_FOR_REMOTE_KV state.
request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
step_skipped_waiting.prepend_request(request)
# Set num_computed_tokens even though KVs are not yet loaded.
# request.num_computed_tokens will not be used anywhere until
# the request finished the KV transfer.
@@ -1286,13 +1279,16 @@ class Scheduler(SchedulerInterface):
# overwrite; the zeroing could race the write.
self._skip_zero_block_ids.update(
self.kv_cache_manager.get_zeroing_block_ids_in_range(
request.request_id,
request_id,
num_new_local_computed_tokens,
num_computed_tokens,
)
)
skip_request(request_queue)
continue
request = request_queue.pop_request()
self.deferred_waiting.discard(request)
self.running.append(request)
if num_external_computed_tokens > 0:
# load_kv_async is False here
@@ -1343,13 +1339,16 @@ class Scheduler(SchedulerInterface):
self.ec_connector.update_state_after_alloc(request, i)
# re-queue requests skipped in this pass ahead of older skipped items.
if step_skipped_kv_holding:
self.kv_holding_waiting.prepend_requests(step_skipped_kv_holding)
if step_skipped_waiting:
self.skipped_waiting.prepend_requests(step_skipped_waiting)
self.waiting.prepend_requests(step_skipped_waiting)
# DP prefill balancing: on a step that admitted prefills (release),
# record whether it was capacity-bound.
if not defer_prefills:
self.prefill_capacity_bound = bool(self.waiting)
_, num_waiting = self.get_request_counts()
self.prefill_capacity_bound = num_waiting > len(self.deferred_waiting)
# Check if the scheduling constraints are satisfied.
total_num_scheduled_tokens = sum(num_scheduled_tokens.values())
@@ -2237,7 +2236,8 @@ class Scheduler(SchedulerInterface):
if stopped_preempted_reqs:
# This is a rare case and unlikely to impact performance.
self.waiting.remove_requests(stopped_preempted_reqs)
self.skipped_waiting.remove_requests(stopped_preempted_reqs)
self.kv_holding_waiting.remove_requests(stopped_preempted_reqs)
self.deferred_waiting.difference_update(stopped_preempted_reqs)
error_req_ids = set(self.grammar_compile_error_reqs)
self.grammar_compile_error_reqs.clear()
@@ -2381,24 +2381,27 @@ class Scheduler(SchedulerInterface):
RequestStatus.WAITING_FOR_STREAMING_REQ,
)
@staticmethod
def _holds_kv_blocks(request: Request) -> bool:
# Whether a request currently has kv blocks allocated.
# While always the case when status is WAITING_FOR_REMOTE_KVS
# or WAITING_FOR_STREAMING_REQ, there are such cases where it's
# WAITING or PREEMPTED.
return (
request.num_computed_tokens > 0
# num_computed_tokens is reset to 0 if kv transfer fails for requests in
# WAITING_FOR_REMOTE_KVS state, before their allocated blocks are freed.
or request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
)
def _enqueue_waiting_request(self, request: Request) -> None:
if self._is_blocked_waiting_status(request.status):
self.skipped_waiting.add_request(request)
self.deferred_waiting.add(request)
if self._holds_kv_blocks(request):
self.kv_holding_waiting.add_request(request)
else:
self.waiting.add_request(request)
def _select_waiting_queue_for_scheduling(self) -> RequestQueue | None:
if self.policy == SchedulingPolicy.FCFS:
return self.skipped_waiting or self.waiting or None
# PRIORITY mode: compare queue heads when both queues are non-empty.
if self.waiting and self.skipped_waiting:
waiting_req = self.waiting.peek_request()
skipped_req = self.skipped_waiting.peek_request()
return self.waiting if waiting_req < skipped_req else self.skipped_waiting
return self.waiting or self.skipped_waiting or None
def _handle_stopped_request(self, request: Request) -> bool:
"""Return True if finished (can be False for resumable requests)."""
if not request.resumable:
@@ -2544,7 +2547,7 @@ class Scheduler(SchedulerInterface):
def get_request_counts(self) -> tuple[int, int]:
"""Returns (num_running_reqs, num_waiting_reqs)."""
return len(self.running), len(self.waiting) + len(self.skipped_waiting)
return len(self.running), len(self.waiting) + len(self.kv_holding_waiting)
def get_kv_cache_usage(self) -> float:
"""Returns the fraction of the KV cache currently in use (0.0-1.0)."""
@@ -2625,7 +2628,8 @@ class Scheduler(SchedulerInterface):
self.running = remove_all(self.running, running_requests_to_remove)
if waiting_requests_to_remove:
self.waiting.remove_requests(waiting_requests_to_remove)
self.skipped_waiting.remove_requests(waiting_requests_to_remove)
self.kv_holding_waiting.remove_requests(waiting_requests_to_remove)
self.deferred_waiting.difference_update(waiting_requests_to_remove)
# Second pass: set status and free requests
for request in valid_requests:
@@ -2733,14 +2737,11 @@ class Scheduler(SchedulerInterface):
def get_num_unfinished_requests(self) -> int:
if self._pause_state == PauseState.PAUSED_ALL:
return 0
num_running, num_waiting = self.get_request_counts()
if self._pause_state == PauseState.PAUSED_NEW:
return len(self.running)
num_waiting = (
len(self.waiting)
+ len(self.skipped_waiting)
- self.num_waiting_for_streaming_input
)
return num_waiting + len(self.running)
return num_running
num_waiting -= self.num_waiting_for_streaming_input
return num_waiting + num_running
def has_finished_requests(self) -> bool:
if self.finished_req_ids:
@@ -2749,10 +2750,8 @@ class Scheduler(SchedulerInterface):
return False
# Finished requests waiting on delayed connector cleanup remain in
# self.requests after they have been removed from scheduling queues.
num_in_queues = (
len(self.waiting) + len(self.skipped_waiting) + len(self.running)
)
return len(self.requests) > num_in_queues
num_running, num_waiting = self.get_request_counts()
return len(self.requests) > (num_running + num_waiting)
def has_requests(self) -> bool:
# Override the interface default to also keep the engine alive while a
@@ -2884,10 +2883,12 @@ class Scheduler(SchedulerInterface):
ec_connector_stats_payload = (
ec_connector_stats.data if ec_connector_stats else None
)
num_running, num_waiting = self.get_request_counts()
num_deferred = len(self.deferred_waiting)
return SchedulerStats(
num_running_reqs=len(self.running),
num_waiting_reqs=len(self.waiting),
num_skipped_waiting_reqs=len(self.skipped_waiting),
num_running_reqs=num_running,
num_waiting_reqs=num_waiting - num_deferred,
num_skipped_waiting_reqs=num_deferred,
kv_cache_usage=self.kv_cache_manager.usage,
prefix_cache_stats=prefix_cache_stats,
connector_prefix_cache_stats=connector_prefix_cache_stats,
@@ -3093,39 +3094,39 @@ class Scheduler(SchedulerInterface):
request.num_computed_tokens = request.num_tokens - 1
self.finished_recving_kv_req_ids.remove(request.request_id)
def _try_promote_blocked_waiting_request(self, request: Request) -> bool:
"""Try to promote a blocked waiting request back to schedulable states."""
def _handle_blocked_waiting_request(self, request: Request) -> bool:
"""Returns True if scheduling of this request should proceed, False if
it should be skipped for this step."""
if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS:
# finished_recving_kv_req_ids is populated during
# update_from_output(), based on worker-side connector signals
# in KVConnectorOutput.finished_recving
if request.request_id not in self.finished_recving_kv_req_ids:
# Check whether async loading requests are finished.
request_id = request.request_id
if request_id not in self.finished_recving_kv_req_ids:
logger.debug("%s is still in WAITING_FOR_REMOTE_KVS state.", request_id)
return False
# Promote request to waiting or preempted.
self._update_waiting_for_remote_kv(request)
if request.num_preemptions:
request.status = RequestStatus.PREEMPTED
else:
request.status = RequestStatus.WAITING
return True
if request.status == RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR:
structured_output_req = request.structured_output_request
if not structured_output_req or structured_output_req.grammar is None:
elif request.status == RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR:
# Check whether grammar compilation has finished.
struct_output_req = request.structured_output_request
if not struct_output_req or struct_output_req.grammar is None:
return False
if isinstance(structured_output_req.grammar, Exception):
if isinstance(struct_output_req.grammar, Exception):
self.grammar_compile_error_reqs.add(request.request_id)
return False
# Grammar is ready, promote to waiting state.
request.status = RequestStatus.WAITING
return True
if request.status == RequestStatus.WAITING_FOR_STREAMING_REQ:
elif request.status == RequestStatus.WAITING_FOR_STREAMING_REQ:
# Streaming input request waiting for more input.
assert not request.streaming_queue
return False
raise AssertionError(
"Unexpected blocked waiting status in promotion: "
f"{request.status.name} for request {request.request_id}"
)
return True
def _update_from_kv_xfer_finished(self, kv_connector_output: KVConnectorOutput):
"""KV Connector: update the scheduler state based on the output.
@@ -3298,7 +3299,7 @@ class Scheduler(SchedulerInterface):
# handle async KV loads (not cached yet, evict_blocks=False)
async_load_reqs = (
req
for req in self.skipped_waiting
for req in self.kv_holding_waiting
if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
)
async_failed_req_ids, num_failed_tokens, _ = (
+3 -2
View File
@@ -188,8 +188,9 @@ class SchedulerStats:
num_running_reqs: int = 0
num_waiting_reqs: int = 0 # length of the "waiting" request queue
num_skipped_waiting_reqs: int = 0 # length of the "skipped waiting" queue
num_waiting_reqs: int = 0 # waiting requests not deferred
# waiting requests deferred by transient constraints or a blocked status
num_skipped_waiting_reqs: int = 0
# These are used for internal DP load-balancing.
step_counter: int = 0