[Core] Rework scheduler skipped_waiting queue (#58947)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
@@ -278,9 +278,9 @@ pub use kv_connector::*;
|
||||
pub struct SchedulerStats {
|
||||
/// Number of requests in model execution batches.
|
||||
pub num_running_reqs: u64,
|
||||
/// Length of the "waiting" request queue.
|
||||
/// Number of waiting requests not deferred.
|
||||
pub num_waiting_reqs: u64,
|
||||
/// Length of the "skipped waiting" queue.
|
||||
/// Number of waiting requests deferred by transient constraints or a blocked status.
|
||||
#[serde(default)]
|
||||
pub num_skipped_waiting_reqs: u64,
|
||||
/// Internal DP load-balancing step counter.
|
||||
|
||||
@@ -2273,7 +2273,8 @@ def test_spec_decode_padding_skipped_with_prefill_in_batch():
|
||||
|
||||
|
||||
def test_scheduler_stats_waiting_queues():
|
||||
"""Test that scheduler stats correctly report waiting and skipped_waiting queues."""
|
||||
"""Test that scheduler stats correctly report capacity-bound and blocked
|
||||
waiting requests."""
|
||||
# Create scheduler with limited capacity so we can have waiting requests
|
||||
scheduler = create_scheduler(max_num_batched_tokens=100)
|
||||
|
||||
@@ -2286,18 +2287,18 @@ def test_scheduler_stats_waiting_queues():
|
||||
for request in all_requests[:3]:
|
||||
scheduler.add_request(request)
|
||||
|
||||
# Manually add 2 more to skipped_waiting to simulate constraint-blocked
|
||||
# Add 2 more that are blocked on grammar compilation
|
||||
for request in all_requests[3:]:
|
||||
request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
scheduler.skipped_waiting.add_request(request)
|
||||
request.status = RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR
|
||||
scheduler.add_request(request)
|
||||
|
||||
# Schedule - this will schedule 2 requests, leaving 1 in waiting
|
||||
output = scheduler.schedule()
|
||||
|
||||
# Verify: 2 scheduled, 1 still waiting on capacity, 2 blocked by constraints
|
||||
assert len(output.scheduled_new_reqs) == 2
|
||||
assert len(scheduler.waiting) == 1
|
||||
assert len(scheduler.skipped_waiting) == 2
|
||||
assert len(scheduler.waiting) == 3
|
||||
assert len(scheduler.deferred_waiting) == 2
|
||||
|
||||
# Call update_from_output() to get frontend-facing stat
|
||||
scheduled_req_ids = list(output.num_scheduled_tokens.keys())
|
||||
@@ -2395,7 +2396,7 @@ def _step_until_done(
|
||||
|
||||
|
||||
def _num_waiting_requests(scheduler: Scheduler) -> int:
|
||||
return len(scheduler.waiting) + len(scheduler.skipped_waiting)
|
||||
return len(scheduler.waiting) + len(scheduler.kv_holding_waiting)
|
||||
|
||||
|
||||
def _step_until_kv_transfer_finished(scheduler: Scheduler, req_ids: list[str]):
|
||||
@@ -2944,7 +2945,7 @@ def test_kv_connector_handles_preemption(
|
||||
if is_async:
|
||||
waiting_req_ids = [
|
||||
req.request_id
|
||||
for req in scheduler.skipped_waiting
|
||||
for req in scheduler.kv_holding_waiting
|
||||
if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
]
|
||||
assert len(waiting_req_ids) == 1
|
||||
@@ -3807,8 +3808,8 @@ def test_schedule_skip_tokenizer_init_structured_output_request():
|
||||
output = scheduler.schedule()
|
||||
assert len(output.scheduled_new_reqs) == 0
|
||||
assert len(scheduler.running) == 0
|
||||
assert len(scheduler.waiting) == 0
|
||||
assert len(scheduler.skipped_waiting) == 1
|
||||
assert len(scheduler.waiting) == 1
|
||||
assert len(scheduler.deferred_waiting) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_grammar", [True, False])
|
||||
@@ -5227,7 +5228,7 @@ def test_prepend_skipped_requests_order():
|
||||
req.status = RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
scheduler.waiting.remove_requests(expected_waiting_reqs[:2])
|
||||
for req in expected_waiting_reqs[:2]:
|
||||
scheduler.skipped_waiting.add_request(req)
|
||||
scheduler.kv_holding_waiting.add_request(req)
|
||||
|
||||
# schedule step
|
||||
# expect the first 2 waiting to be skipped, the third running,
|
||||
@@ -5237,21 +5238,18 @@ def test_prepend_skipped_requests_order():
|
||||
# pop the third request which is expected to be running
|
||||
expected_waiting_reqs.pop(2)
|
||||
|
||||
# verify waiting order is preserved
|
||||
waiting_reqs = list(scheduler.skipped_waiting) + list(scheduler.waiting)
|
||||
# verify waiting order is preserved, with the KV-holding requests first
|
||||
waiting_reqs = list(scheduler.kv_holding_waiting) + list(scheduler.waiting)
|
||||
assert waiting_reqs == expected_waiting_reqs
|
||||
|
||||
|
||||
def test_remote_kv_promotion_keeps_fcfs_with_grammar_prefix():
|
||||
def test_remote_kv_promotion_drains_before_grammar_prefix():
|
||||
scheduler = create_scheduler(max_num_seqs=1)
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_num_new_matched_tokens.return_value = (0, False)
|
||||
|
||||
requests = create_requests(num_requests=4)
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
|
||||
req_grammar_1, req_grammar_2, req_remote, req_tail = list(scheduler.waiting)
|
||||
req_grammar_1, req_grammar_2, req_remote, req_tail = requests
|
||||
|
||||
# simulate two structured-output grammar requests at the waiting head
|
||||
# that become ready now.
|
||||
@@ -5260,31 +5258,28 @@ def test_remote_kv_promotion_keeps_fcfs_with_grammar_prefix():
|
||||
req_grammar_2.status = RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR
|
||||
req_grammar_2.structured_output_request = Mock(grammar=object())
|
||||
|
||||
# simulate a remote-KV request that is ready to be promoted now.
|
||||
# simulate a remote-KV request that is ready to be promoted now. Parked
|
||||
# loads hold KV blocks, so they drain ahead of the grammar requests.
|
||||
req_remote.status = RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
scheduler.waiting.remove_requests([req_grammar_1, req_grammar_2, req_remote])
|
||||
scheduler.skipped_waiting.add_request(req_grammar_1)
|
||||
scheduler.skipped_waiting.add_request(req_grammar_2)
|
||||
scheduler.skipped_waiting.add_request(req_remote)
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
assert list(scheduler.kv_holding_waiting) == [req_remote]
|
||||
scheduler.finished_recving_kv_req_ids.add(req_remote.request_id)
|
||||
scheduler._update_waiting_for_remote_kv = Mock()
|
||||
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert output.scheduled_new_reqs
|
||||
assert output.scheduled_new_reqs[0].req_id == req_grammar_1.request_id
|
||||
waiting_req_ids = [
|
||||
req.request_id
|
||||
for req in list(scheduler.skipped_waiting) + list(scheduler.waiting)
|
||||
]
|
||||
assert output.scheduled_new_reqs[0].req_id == req_remote.request_id
|
||||
waiting_req_ids = [req.request_id for req in scheduler.waiting]
|
||||
assert waiting_req_ids == [
|
||||
req_grammar_1.request_id,
|
||||
req_grammar_2.request_id,
|
||||
req_remote.request_id,
|
||||
req_tail.request_id,
|
||||
]
|
||||
|
||||
|
||||
def test_fcfs_mixed_skipped_waiting_types_keep_order():
|
||||
def test_fcfs_mixed_blocked_types_remote_load_drains_first():
|
||||
scheduler = create_scheduler(max_num_batched_tokens=20)
|
||||
scheduler._update_waiting_for_remote_kv = Mock()
|
||||
|
||||
@@ -5305,7 +5300,8 @@ def test_fcfs_mixed_skipped_waiting_types_keep_order():
|
||||
for req in (req_grammar, req_remote, req_stream, req_regular, req_tail):
|
||||
scheduler.add_request(req)
|
||||
scheduler.schedule()
|
||||
assert list(scheduler.skipped_waiting) == [req_grammar, req_remote, req_stream]
|
||||
assert list(scheduler.waiting) == [req_grammar, req_stream, req_tail]
|
||||
assert list(scheduler.kv_holding_waiting) == [req_remote]
|
||||
|
||||
scheduler.finish_requests(req_regular.request_id, RequestStatus.FINISHED_ABORTED)
|
||||
assert not scheduler.running
|
||||
@@ -5316,8 +5312,8 @@ def test_fcfs_mixed_skipped_waiting_types_keep_order():
|
||||
|
||||
second_output = scheduler.schedule()
|
||||
expected_order = [
|
||||
req_grammar.request_id,
|
||||
req_remote.request_id,
|
||||
req_grammar.request_id,
|
||||
req_stream.request_id,
|
||||
req_tail.request_id,
|
||||
]
|
||||
|
||||
@@ -105,8 +105,8 @@ def test_streaming_pause_does_not_over_admit_worker_slot():
|
||||
# still holds its slot.
|
||||
scheduler.update_from_output(out, _model_runner_output(["session"], [[STOP_TOKEN]]))
|
||||
assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
# A blocked waiting status lands in skipped_waiting, not the main queue.
|
||||
assert session in scheduler.skipped_waiting
|
||||
# The paused session keeps its KV blocks, so it lands in kv_holding_waiting.
|
||||
assert session in scheduler.kv_holding_waiting
|
||||
assert scheduler.num_waiting_for_streaming_input == 1
|
||||
|
||||
# A different request arrives while the session is paused.
|
||||
|
||||
@@ -119,7 +119,7 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler):
|
||||
|
||||
scheduler_output = fail_scheduler.schedule()
|
||||
|
||||
assert len(fail_scheduler.skipped_waiting) == 1
|
||||
assert len(fail_scheduler.kv_holding_waiting) == 1
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert request.num_computed_tokens == num_external_computed_tokens
|
||||
|
||||
@@ -145,4 +145,5 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler):
|
||||
assert output.finish_reason == FinishReason.ERROR
|
||||
|
||||
assert len(fail_scheduler.waiting) == 0
|
||||
assert len(fail_scheduler.skipped_waiting) == 0
|
||||
assert len(fail_scheduler.kv_holding_waiting) == 0
|
||||
assert not fail_scheduler.deferred_waiting
|
||||
|
||||
@@ -333,7 +333,7 @@ def test_async_recompute_blocks_not_cached_when_invalid(
|
||||
scheduler_output = recompute_scheduler.schedule()
|
||||
|
||||
# request should be waiting for remote KVs
|
||||
assert len(recompute_scheduler.skipped_waiting) == 1
|
||||
assert len(recompute_scheduler.kv_holding_waiting) == 1
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert request.num_computed_tokens == num_external_computed_tokens
|
||||
|
||||
|
||||
@@ -111,8 +111,8 @@ def test_async_load_failure(
|
||||
scheduler_output = scheduler.schedule()
|
||||
|
||||
assert len(scheduler.waiting) == 0
|
||||
assert len(scheduler.skipped_waiting) == 3
|
||||
for request in scheduler.skipped_waiting:
|
||||
assert len(scheduler.kv_holding_waiting) == 3
|
||||
for request in scheduler.kv_holding_waiting:
|
||||
assert request.num_computed_tokens == num_external_computed_tokens
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert scheduler.connector.get_num_new_matched_tokens.call_count == 3
|
||||
@@ -132,8 +132,8 @@ def test_async_load_failure(
|
||||
min_invalid_block_idx = min(invalid_block_idxs)
|
||||
|
||||
assert len(scheduler.waiting) == 0
|
||||
assert len(scheduler.skipped_waiting) == 3
|
||||
for request in scheduler.skipped_waiting:
|
||||
assert len(scheduler.kv_holding_waiting) == 3
|
||||
for request in scheduler.kv_holding_waiting:
|
||||
if request.request_id == request2.request_id:
|
||||
assert request.num_computed_tokens == (
|
||||
min_invalid_block_idx * scheduler.block_size
|
||||
@@ -340,8 +340,9 @@ def test_async_progressive_load_failure(
|
||||
scheduler_output = scheduler.schedule()
|
||||
|
||||
assert len(scheduler.waiting) == 0
|
||||
assert len(scheduler.skipped_waiting) == 1
|
||||
assert scheduler.skipped_waiting.peek_request().request_id == request.request_id
|
||||
assert len(scheduler.kv_holding_waiting) == 1
|
||||
peeked = scheduler.kv_holding_waiting.peek_request()
|
||||
assert peeked.request_id == request.request_id
|
||||
assert request.num_computed_tokens == num_external_computed_tokens
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert scheduler.connector.get_num_new_matched_tokens.call_count == 1
|
||||
@@ -363,8 +364,9 @@ def test_async_progressive_load_failure(
|
||||
min_invalid_block_idx = min(min_invalid_block_idx, invalid_block_idx)
|
||||
|
||||
assert len(scheduler.waiting) == 0
|
||||
assert len(scheduler.skipped_waiting) == 1
|
||||
assert scheduler.skipped_waiting.peek_request().request_id == request.request_id
|
||||
assert len(scheduler.kv_holding_waiting) == 1
|
||||
peeked = scheduler.kv_holding_waiting.peek_request()
|
||||
assert peeked.request_id == request.request_id
|
||||
assert request.num_computed_tokens == (
|
||||
min_invalid_block_idx * scheduler.block_size
|
||||
)
|
||||
|
||||
@@ -3026,7 +3026,7 @@ def test_empty_recv_is_reported_only_when_awaited(
|
||||
request is named in neither _recving_transfers nor _failed_recv_reqs and
|
||||
never reaches finished_recving. The scheduler has no other way to release a
|
||||
WAITING_FOR_REMOTE_KVS request, and there is no timeout, so it sits in
|
||||
skipped_waiting holding its blocks for the life of the process.
|
||||
kv_holding_waiting holding its blocks for the life of the process.
|
||||
|
||||
Notify-only (awaiting_kvs=False): request_finished seeding an empty recv to
|
||||
free P's blocks for a request aborted before it was scheduled, or a readback
|
||||
|
||||
@@ -25,7 +25,7 @@ pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def _num_waiting_requests(scheduler) -> int:
|
||||
return len(scheduler.waiting) + len(scheduler.skipped_waiting)
|
||||
return len(scheduler.waiting) + len(scheduler.kv_holding_waiting)
|
||||
|
||||
|
||||
def test_basic_lifecycle():
|
||||
@@ -64,7 +64,7 @@ def test_basic_lifecycle():
|
||||
|
||||
# Req waiting for KVs with no computed/scheduled toks ...
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
assert request in scheduler.skipped_waiting
|
||||
assert request in scheduler.kv_holding_waiting
|
||||
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert request.num_computed_tokens == NUM_TOKENS
|
||||
|
||||
@@ -791,3 +791,71 @@ def test_async_load_reserves_blocks_for_promotion_margin():
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert req_a.status == RequestStatus.RUNNING
|
||||
assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0
|
||||
|
||||
|
||||
def test_deferred_lookup_does_not_block_parked_load():
|
||||
"""A request with a deferred connector lookup must not strand an async
|
||||
load queued behind it.
|
||||
|
||||
req_a's lookup is deferred (connector returns None), so it is parked in
|
||||
waiting holding no blocks; req_b is then admitted as an async
|
||||
load holding most of the pool. Once req_a's lookup resolves it cannot
|
||||
allocate, and the waiting scan stops at it every step. The parked load
|
||||
must still be promotable: it lives in the KV-holding queue
|
||||
that is drained first, so it runs, finishes, and frees its blocks for
|
||||
req_a.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7
|
||||
|
||||
req_a = create_request(
|
||||
request_id=1, block_size=BLOCK_SIZE, num_tokens=BLOCK_SIZE * 4, max_tokens=1
|
||||
)
|
||||
req_b = create_request(
|
||||
request_id=2,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=BLOCK_SIZE * 5,
|
||||
do_remote_prefill=True,
|
||||
max_tokens=1,
|
||||
)
|
||||
scheduler.add_request(req_a)
|
||||
scheduler.add_request(req_b)
|
||||
|
||||
deferrals = [(None, False)]
|
||||
|
||||
def lookup(request, num_computed_tokens):
|
||||
if request is req_b:
|
||||
return BLOCK_SIZE * 5, True
|
||||
return deferrals.pop() if deferrals else (0, False)
|
||||
|
||||
with patch.object(
|
||||
scheduler.connector, "get_num_new_matched_tokens", side_effect=lookup
|
||||
):
|
||||
# Step 1: req_a's lookup is deferred; req_b is admitted as an async
|
||||
# load holding 5 of 7 usable blocks.
|
||||
scheduler.schedule()
|
||||
assert req_a.status == RequestStatus.WAITING
|
||||
assert req_b.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
|
||||
# req_a's lookup resolves (no hit) while req_b's transfer is in
|
||||
# flight; req_a needs 4 blocks but only 2 are free, so the waiting
|
||||
# scan stops at it. req_b's transfer then finishes.
|
||||
scheduler.update_from_output(
|
||||
scheduler.schedule(),
|
||||
create_model_runner_output([], finished_recving={req_b.request_id}),
|
||||
)
|
||||
|
||||
# req_b must still be reached, promoted, and scheduled despite the
|
||||
# scan stopping at req_a.
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert req_b.status == RequestStatus.RUNNING
|
||||
assert scheduler_output.num_scheduled_tokens[req_b.request_id] > 0
|
||||
|
||||
# req_b finishes and frees its blocks; now req_a fits and runs.
|
||||
scheduler.update_from_output(
|
||||
scheduler_output, create_model_runner_output([req_b])
|
||||
)
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert req_a.status == RequestStatus.RUNNING
|
||||
assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0
|
||||
|
||||
@@ -54,6 +54,8 @@ def assert_scheduler_empty(scheduler: Scheduler):
|
||||
# Scheduler Metadata.
|
||||
assert len(scheduler.requests) == 0
|
||||
assert len(scheduler.waiting) == 0
|
||||
assert len(scheduler.kv_holding_waiting) == 0
|
||||
assert not scheduler.deferred_waiting
|
||||
assert len(scheduler.running) == 0
|
||||
assert len(scheduler.finished_req_ids) == 0
|
||||
assert len(scheduler.finished_recving_kv_req_ids) == 0
|
||||
|
||||
@@ -554,9 +554,9 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
eco_cycle2 = eco_dict_cycle2[session.client_index].outputs[0]
|
||||
assert eco_cycle2.finish_reason == FinishReason.STOP
|
||||
assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
# Sessions paused for streaming input are blocked-waiting, so they
|
||||
# live in the skipped_waiting queue rather than the main waiting queue.
|
||||
assert session in scheduler.skipped_waiting
|
||||
# Sessions paused for streaming input keep their KV blocks, so they
|
||||
# live in the kv_holding_waiting queue.
|
||||
assert session in scheduler.kv_holding_waiting
|
||||
assert session._all_token_ids == [1, 2, 3, 10, STOP_TOKEN]
|
||||
|
||||
# CRITICAL ASSERTION: Cached prompt_token_ids STILL must not have changed
|
||||
|
||||
@@ -41,7 +41,7 @@ class RequestQueue(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def prepend_requests(self, requests: "RequestQueue") -> None:
|
||||
def prepend_requests(self, requests: Iterable[Request]) -> None:
|
||||
"""Prepend all requests from another queue to the front of this
|
||||
queue."""
|
||||
pass
|
||||
@@ -93,7 +93,7 @@ class FCFSRequestQueue(deque[Request], RequestQueue):
|
||||
"""Prepend a request to the front of the queue."""
|
||||
self.appendleft(request)
|
||||
|
||||
def prepend_requests(self, requests: RequestQueue) -> None:
|
||||
def prepend_requests(self, requests: Iterable[Request]) -> None:
|
||||
"""Prepend all requests from another queue to the front of this
|
||||
queue.
|
||||
|
||||
@@ -163,7 +163,7 @@ class PriorityRequestQueue(RequestQueue):
|
||||
front. Requests are ordered by (priority, arrival_time)."""
|
||||
self.add_request(request)
|
||||
|
||||
def prepend_requests(self, requests: RequestQueue) -> None:
|
||||
def prepend_requests(self, requests: Iterable[Request]) -> None:
|
||||
"""Add all requests from another queue according to priority policy.
|
||||
|
||||
Note: In a priority queue, there is no concept of prepending to the
|
||||
|
||||
@@ -210,8 +210,13 @@ class Scheduler(SchedulerInterface):
|
||||
) from e
|
||||
# Priority queues for requests.
|
||||
self.waiting = create_request_queue(self.policy)
|
||||
# requests skipped in waiting flow due async deps or constraints.
|
||||
self.skipped_waiting = create_request_queue(self.policy)
|
||||
# Waiting requests that hold KV blocks are always drained before
|
||||
# self.waiting: a block-holder must never sit behind a request whose
|
||||
# failed allocation would stop the scheduling scan.
|
||||
self.kv_holding_waiting = create_request_queue(self.policy)
|
||||
# Waiting requests that are deferred, i.e. were skipped by the scheduling
|
||||
# scan or enqueued in a blocked status.
|
||||
self.deferred_waiting: set[Request] = set()
|
||||
self.running: list[Request] = []
|
||||
|
||||
# The request IDs that are finished in between the previous and the
|
||||
@@ -606,9 +611,8 @@ class Scheduler(SchedulerInterface):
|
||||
# `long_prefill_token_threshold` exists to stop a long prefill from
|
||||
# starving other requests of the token budget. When it is the only
|
||||
# request there is nobody to starve, so let it use the whole budget.
|
||||
num_eligible_reqs = (
|
||||
len(self.running) + len(self.waiting) + len(self.skipped_waiting)
|
||||
)
|
||||
num_running, num_waiting = self.get_request_counts()
|
||||
num_eligible_reqs = num_running + num_waiting
|
||||
long_prefill_token_threshold = (
|
||||
self.scheduler_config.long_prefill_token_threshold
|
||||
if num_eligible_reqs > 1
|
||||
@@ -867,10 +871,21 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
# Next, schedule the WAITING requests.
|
||||
if not preempted_reqs and self._pause_state == PauseState.UNPAUSED:
|
||||
step_skipped_waiting = create_request_queue(self.policy)
|
||||
step_skipped_waiting: deque[Request] = deque()
|
||||
step_skipped_kv_holding: deque[Request] = deque()
|
||||
|
||||
while (self.waiting or self.skipped_waiting) and token_budget > 0:
|
||||
if input_budget <= draft_slots:
|
||||
def skip_request(from_queue: RequestQueue) -> None:
|
||||
request_to_skip = from_queue.pop_request()
|
||||
self.deferred_waiting.add(request_to_skip)
|
||||
if self._holds_kv_blocks(request_to_skip):
|
||||
step_skipped_kv_holding.appendleft(request_to_skip)
|
||||
else:
|
||||
step_skipped_waiting.appendleft(request_to_skip)
|
||||
|
||||
while token_budget > 0:
|
||||
# Requests holding KV blocks are always drained first.
|
||||
request_queue = self.kv_holding_waiting or self.waiting
|
||||
if not request_queue or input_budget <= draft_slots:
|
||||
break
|
||||
# Paused streaming sessions (WAITING_FOR_STREAMING_REQ) are not
|
||||
# in `running` but still hold a model-runner request slot.
|
||||
@@ -878,38 +893,22 @@ class Scheduler(SchedulerInterface):
|
||||
if num_running >= self.max_num_active_reqs:
|
||||
break
|
||||
|
||||
request_queue = self._select_waiting_queue_for_scheduling()
|
||||
assert request_queue is not None
|
||||
|
||||
request = request_queue.peek_request()
|
||||
request_id = request.request_id
|
||||
|
||||
# try to promote blocked statuses while traversing skipped queue.
|
||||
if self._is_blocked_waiting_status(
|
||||
request.status
|
||||
) and not self._try_promote_blocked_waiting_request(request):
|
||||
if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS:
|
||||
logger.debug(
|
||||
"%s is still in WAITING_FOR_REMOTE_KVS state.",
|
||||
request_id,
|
||||
)
|
||||
request_queue.pop_request()
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
ready_to_schedule = self._handle_blocked_waiting_request(request)
|
||||
if not ready_to_schedule:
|
||||
skip_request(request_queue)
|
||||
continue
|
||||
|
||||
if (
|
||||
request.num_stale_output_tokens > 0
|
||||
and not request.drop_stale_output
|
||||
):
|
||||
if request.num_stale_output_tokens and not request.drop_stale_output:
|
||||
# Deliverable stale output still in flight: resuming now
|
||||
# could resample a position that output later delivers.
|
||||
# It drains within the pipeline depth.
|
||||
request_queue.pop_request()
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
skip_request(request_queue)
|
||||
continue
|
||||
|
||||
# Check that adding the request still respects the max_loras
|
||||
# constraint.
|
||||
# Check that adding the request still respects the max_loras constraint.
|
||||
if (
|
||||
self.lora_config
|
||||
and request.lora_request
|
||||
@@ -919,8 +918,7 @@ class Scheduler(SchedulerInterface):
|
||||
)
|
||||
):
|
||||
# Scheduling would exceed max_loras, skip.
|
||||
request_queue.pop_request()
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
skip_request(request_queue)
|
||||
continue
|
||||
|
||||
num_external_computed_tokens = 0
|
||||
@@ -959,8 +957,7 @@ class Scheduler(SchedulerInterface):
|
||||
# The request cannot be scheduled because
|
||||
# the KVConnector couldn't determine
|
||||
# the number of matched tokens.
|
||||
request_queue.pop_request()
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
skip_request(request_queue)
|
||||
continue
|
||||
|
||||
if self.prefix_replay_tokens:
|
||||
@@ -1030,8 +1027,7 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
# Skip request with pending mm encoding prefetches
|
||||
if self._ec_transfer_pending(request, num_computed_tokens):
|
||||
request_queue.pop_request()
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
skip_request(request_queue)
|
||||
continue
|
||||
|
||||
# Track first scheduled prefill, not post-preemption repeat prefills
|
||||
@@ -1052,8 +1048,7 @@ class Scheduler(SchedulerInterface):
|
||||
num_computed_tokens = request.num_computed_tokens
|
||||
|
||||
if self._ec_transfer_pending(request, num_computed_tokens):
|
||||
request_queue.pop_request()
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
skip_request(request_queue)
|
||||
continue
|
||||
|
||||
encoder_inputs_to_schedule = None
|
||||
@@ -1260,12 +1255,10 @@ class Scheduler(SchedulerInterface):
|
||||
request, num_new_local_computed_tokens
|
||||
)
|
||||
|
||||
request = request_queue.pop_request()
|
||||
if load_kv_async:
|
||||
# If loading async, allocate memory and put request
|
||||
# into the WAITING_FOR_REMOTE_KV state.
|
||||
request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
step_skipped_waiting.prepend_request(request)
|
||||
# Set num_computed_tokens even though KVs are not yet loaded.
|
||||
# request.num_computed_tokens will not be used anywhere until
|
||||
# the request finished the KV transfer.
|
||||
@@ -1286,13 +1279,16 @@ class Scheduler(SchedulerInterface):
|
||||
# overwrite; the zeroing could race the write.
|
||||
self._skip_zero_block_ids.update(
|
||||
self.kv_cache_manager.get_zeroing_block_ids_in_range(
|
||||
request.request_id,
|
||||
request_id,
|
||||
num_new_local_computed_tokens,
|
||||
num_computed_tokens,
|
||||
)
|
||||
)
|
||||
skip_request(request_queue)
|
||||
continue
|
||||
|
||||
request = request_queue.pop_request()
|
||||
self.deferred_waiting.discard(request)
|
||||
self.running.append(request)
|
||||
if num_external_computed_tokens > 0:
|
||||
# load_kv_async is False here
|
||||
@@ -1343,13 +1339,16 @@ class Scheduler(SchedulerInterface):
|
||||
self.ec_connector.update_state_after_alloc(request, i)
|
||||
|
||||
# re-queue requests skipped in this pass ahead of older skipped items.
|
||||
if step_skipped_kv_holding:
|
||||
self.kv_holding_waiting.prepend_requests(step_skipped_kv_holding)
|
||||
if step_skipped_waiting:
|
||||
self.skipped_waiting.prepend_requests(step_skipped_waiting)
|
||||
self.waiting.prepend_requests(step_skipped_waiting)
|
||||
|
||||
# DP prefill balancing: on a step that admitted prefills (release),
|
||||
# record whether it was capacity-bound.
|
||||
if not defer_prefills:
|
||||
self.prefill_capacity_bound = bool(self.waiting)
|
||||
_, num_waiting = self.get_request_counts()
|
||||
self.prefill_capacity_bound = num_waiting > len(self.deferred_waiting)
|
||||
|
||||
# Check if the scheduling constraints are satisfied.
|
||||
total_num_scheduled_tokens = sum(num_scheduled_tokens.values())
|
||||
@@ -2237,7 +2236,8 @@ class Scheduler(SchedulerInterface):
|
||||
if stopped_preempted_reqs:
|
||||
# This is a rare case and unlikely to impact performance.
|
||||
self.waiting.remove_requests(stopped_preempted_reqs)
|
||||
self.skipped_waiting.remove_requests(stopped_preempted_reqs)
|
||||
self.kv_holding_waiting.remove_requests(stopped_preempted_reqs)
|
||||
self.deferred_waiting.difference_update(stopped_preempted_reqs)
|
||||
|
||||
error_req_ids = set(self.grammar_compile_error_reqs)
|
||||
self.grammar_compile_error_reqs.clear()
|
||||
@@ -2381,24 +2381,27 @@ class Scheduler(SchedulerInterface):
|
||||
RequestStatus.WAITING_FOR_STREAMING_REQ,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _holds_kv_blocks(request: Request) -> bool:
|
||||
# Whether a request currently has kv blocks allocated.
|
||||
# While always the case when status is WAITING_FOR_REMOTE_KVS
|
||||
# or WAITING_FOR_STREAMING_REQ, there are such cases where it's
|
||||
# WAITING or PREEMPTED.
|
||||
return (
|
||||
request.num_computed_tokens > 0
|
||||
# num_computed_tokens is reset to 0 if kv transfer fails for requests in
|
||||
# WAITING_FOR_REMOTE_KVS state, before their allocated blocks are freed.
|
||||
or request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
)
|
||||
|
||||
def _enqueue_waiting_request(self, request: Request) -> None:
|
||||
if self._is_blocked_waiting_status(request.status):
|
||||
self.skipped_waiting.add_request(request)
|
||||
self.deferred_waiting.add(request)
|
||||
if self._holds_kv_blocks(request):
|
||||
self.kv_holding_waiting.add_request(request)
|
||||
else:
|
||||
self.waiting.add_request(request)
|
||||
|
||||
def _select_waiting_queue_for_scheduling(self) -> RequestQueue | None:
|
||||
if self.policy == SchedulingPolicy.FCFS:
|
||||
return self.skipped_waiting or self.waiting or None
|
||||
|
||||
# PRIORITY mode: compare queue heads when both queues are non-empty.
|
||||
if self.waiting and self.skipped_waiting:
|
||||
waiting_req = self.waiting.peek_request()
|
||||
skipped_req = self.skipped_waiting.peek_request()
|
||||
return self.waiting if waiting_req < skipped_req else self.skipped_waiting
|
||||
|
||||
return self.waiting or self.skipped_waiting or None
|
||||
|
||||
def _handle_stopped_request(self, request: Request) -> bool:
|
||||
"""Return True if finished (can be False for resumable requests)."""
|
||||
if not request.resumable:
|
||||
@@ -2544,7 +2547,7 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
def get_request_counts(self) -> tuple[int, int]:
|
||||
"""Returns (num_running_reqs, num_waiting_reqs)."""
|
||||
return len(self.running), len(self.waiting) + len(self.skipped_waiting)
|
||||
return len(self.running), len(self.waiting) + len(self.kv_holding_waiting)
|
||||
|
||||
def get_kv_cache_usage(self) -> float:
|
||||
"""Returns the fraction of the KV cache currently in use (0.0-1.0)."""
|
||||
@@ -2625,7 +2628,8 @@ class Scheduler(SchedulerInterface):
|
||||
self.running = remove_all(self.running, running_requests_to_remove)
|
||||
if waiting_requests_to_remove:
|
||||
self.waiting.remove_requests(waiting_requests_to_remove)
|
||||
self.skipped_waiting.remove_requests(waiting_requests_to_remove)
|
||||
self.kv_holding_waiting.remove_requests(waiting_requests_to_remove)
|
||||
self.deferred_waiting.difference_update(waiting_requests_to_remove)
|
||||
|
||||
# Second pass: set status and free requests
|
||||
for request in valid_requests:
|
||||
@@ -2733,14 +2737,11 @@ class Scheduler(SchedulerInterface):
|
||||
def get_num_unfinished_requests(self) -> int:
|
||||
if self._pause_state == PauseState.PAUSED_ALL:
|
||||
return 0
|
||||
num_running, num_waiting = self.get_request_counts()
|
||||
if self._pause_state == PauseState.PAUSED_NEW:
|
||||
return len(self.running)
|
||||
num_waiting = (
|
||||
len(self.waiting)
|
||||
+ len(self.skipped_waiting)
|
||||
- self.num_waiting_for_streaming_input
|
||||
)
|
||||
return num_waiting + len(self.running)
|
||||
return num_running
|
||||
num_waiting -= self.num_waiting_for_streaming_input
|
||||
return num_waiting + num_running
|
||||
|
||||
def has_finished_requests(self) -> bool:
|
||||
if self.finished_req_ids:
|
||||
@@ -2749,10 +2750,8 @@ class Scheduler(SchedulerInterface):
|
||||
return False
|
||||
# Finished requests waiting on delayed connector cleanup remain in
|
||||
# self.requests after they have been removed from scheduling queues.
|
||||
num_in_queues = (
|
||||
len(self.waiting) + len(self.skipped_waiting) + len(self.running)
|
||||
)
|
||||
return len(self.requests) > num_in_queues
|
||||
num_running, num_waiting = self.get_request_counts()
|
||||
return len(self.requests) > (num_running + num_waiting)
|
||||
|
||||
def has_requests(self) -> bool:
|
||||
# Override the interface default to also keep the engine alive while a
|
||||
@@ -2884,10 +2883,12 @@ class Scheduler(SchedulerInterface):
|
||||
ec_connector_stats_payload = (
|
||||
ec_connector_stats.data if ec_connector_stats else None
|
||||
)
|
||||
num_running, num_waiting = self.get_request_counts()
|
||||
num_deferred = len(self.deferred_waiting)
|
||||
return SchedulerStats(
|
||||
num_running_reqs=len(self.running),
|
||||
num_waiting_reqs=len(self.waiting),
|
||||
num_skipped_waiting_reqs=len(self.skipped_waiting),
|
||||
num_running_reqs=num_running,
|
||||
num_waiting_reqs=num_waiting - num_deferred,
|
||||
num_skipped_waiting_reqs=num_deferred,
|
||||
kv_cache_usage=self.kv_cache_manager.usage,
|
||||
prefix_cache_stats=prefix_cache_stats,
|
||||
connector_prefix_cache_stats=connector_prefix_cache_stats,
|
||||
@@ -3093,39 +3094,39 @@ class Scheduler(SchedulerInterface):
|
||||
request.num_computed_tokens = request.num_tokens - 1
|
||||
self.finished_recving_kv_req_ids.remove(request.request_id)
|
||||
|
||||
def _try_promote_blocked_waiting_request(self, request: Request) -> bool:
|
||||
"""Try to promote a blocked waiting request back to schedulable states."""
|
||||
def _handle_blocked_waiting_request(self, request: Request) -> bool:
|
||||
"""Returns True if scheduling of this request should proceed, False if
|
||||
it should be skipped for this step."""
|
||||
if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS:
|
||||
# finished_recving_kv_req_ids is populated during
|
||||
# update_from_output(), based on worker-side connector signals
|
||||
# in KVConnectorOutput.finished_recving
|
||||
if request.request_id not in self.finished_recving_kv_req_ids:
|
||||
# Check whether async loading requests are finished.
|
||||
request_id = request.request_id
|
||||
if request_id not in self.finished_recving_kv_req_ids:
|
||||
logger.debug("%s is still in WAITING_FOR_REMOTE_KVS state.", request_id)
|
||||
return False
|
||||
# Promote request to waiting or preempted.
|
||||
self._update_waiting_for_remote_kv(request)
|
||||
if request.num_preemptions:
|
||||
request.status = RequestStatus.PREEMPTED
|
||||
else:
|
||||
request.status = RequestStatus.WAITING
|
||||
return True
|
||||
|
||||
if request.status == RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR:
|
||||
structured_output_req = request.structured_output_request
|
||||
if not structured_output_req or structured_output_req.grammar is None:
|
||||
elif request.status == RequestStatus.WAITING_FOR_STRUCTURED_OUTPUT_GRAMMAR:
|
||||
# Check whether grammar compilation has finished.
|
||||
struct_output_req = request.structured_output_request
|
||||
if not struct_output_req or struct_output_req.grammar is None:
|
||||
return False
|
||||
if isinstance(structured_output_req.grammar, Exception):
|
||||
if isinstance(struct_output_req.grammar, Exception):
|
||||
self.grammar_compile_error_reqs.add(request.request_id)
|
||||
return False
|
||||
# Grammar is ready, promote to waiting state.
|
||||
request.status = RequestStatus.WAITING
|
||||
return True
|
||||
|
||||
if request.status == RequestStatus.WAITING_FOR_STREAMING_REQ:
|
||||
elif request.status == RequestStatus.WAITING_FOR_STREAMING_REQ:
|
||||
# Streaming input request waiting for more input.
|
||||
assert not request.streaming_queue
|
||||
return False
|
||||
|
||||
raise AssertionError(
|
||||
"Unexpected blocked waiting status in promotion: "
|
||||
f"{request.status.name} for request {request.request_id}"
|
||||
)
|
||||
return True
|
||||
|
||||
def _update_from_kv_xfer_finished(self, kv_connector_output: KVConnectorOutput):
|
||||
"""KV Connector: update the scheduler state based on the output.
|
||||
@@ -3298,7 +3299,7 @@ class Scheduler(SchedulerInterface):
|
||||
# handle async KV loads (not cached yet, evict_blocks=False)
|
||||
async_load_reqs = (
|
||||
req
|
||||
for req in self.skipped_waiting
|
||||
for req in self.kv_holding_waiting
|
||||
if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
)
|
||||
async_failed_req_ids, num_failed_tokens, _ = (
|
||||
|
||||
@@ -188,8 +188,9 @@ class SchedulerStats:
|
||||
|
||||
num_running_reqs: int = 0
|
||||
|
||||
num_waiting_reqs: int = 0 # length of the "waiting" request queue
|
||||
num_skipped_waiting_reqs: int = 0 # length of the "skipped waiting" queue
|
||||
num_waiting_reqs: int = 0 # waiting requests not deferred
|
||||
# waiting requests deferred by transient constraints or a blocked status
|
||||
num_skipped_waiting_reqs: int = 0
|
||||
|
||||
# These are used for internal DP load-balancing.
|
||||
step_counter: int = 0
|
||||
|
||||
Reference in New Issue
Block a user