[Core][Metrics] expose waiting request breakdown via labeled metric (capacity/deferred) (#38435)
Signed-off-by: Mukesh Baphna <mukesh@hippocraticai.com> Signed-off-by: Mark McLoughlin <markmc@redhat.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
Mark McLoughlin
parent
f72b20976c
commit
5c18b961d6
@@ -182,6 +182,7 @@ async def test_metrics_counts(
|
||||
EXPECTED_METRICS_V1 = [
|
||||
"vllm:num_requests_running",
|
||||
"vllm:num_requests_waiting",
|
||||
"vllm:num_requests_waiting_by_reason",
|
||||
"vllm:kv_cache_usage_perc",
|
||||
"vllm:prefix_cache_queries",
|
||||
"vllm:prefix_cache_hits",
|
||||
|
||||
@@ -1039,6 +1039,54 @@ def test_no_spec_tokens_scheduled_for_prefill_chunks():
|
||||
assert len(output.scheduled_spec_decode_tokens[req.request_id]) == num_spec_tokens
|
||||
|
||||
|
||||
def test_scheduler_stats_waiting_queues():
|
||||
"""Test that scheduler stats correctly report waiting and skipped_waiting queues."""
|
||||
# Create scheduler with limited capacity so we can have waiting requests
|
||||
scheduler = create_scheduler(max_num_batched_tokens=100)
|
||||
|
||||
# Create requests: some will be scheduled, some will wait on capacity,
|
||||
# and some will be blocked by constraints
|
||||
all_requests = create_requests(num_requests=5, num_tokens=50)
|
||||
|
||||
# Add 3 requests - only 2 can be scheduled (2 * 50 = 100 tokens)
|
||||
# The 3rd will remain in waiting queue (capacity constraint)
|
||||
for request in all_requests[:3]:
|
||||
scheduler.add_request(request)
|
||||
|
||||
# Manually add 2 more to skipped_waiting to simulate constraint-blocked
|
||||
for request in all_requests[3:]:
|
||||
request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
scheduler.skipped_waiting.add_request(request)
|
||||
|
||||
# Schedule - this will schedule 2 requests, leaving 1 in waiting
|
||||
output = scheduler.schedule()
|
||||
|
||||
# Verify: 2 scheduled, 1 still waiting on capacity, 2 blocked by constraints
|
||||
assert len(output.scheduled_new_reqs) == 2
|
||||
assert len(scheduler.waiting) == 1
|
||||
assert len(scheduler.skipped_waiting) == 2
|
||||
|
||||
# Call update_from_output() to get frontend-facing stat
|
||||
scheduled_req_ids = list(output.num_scheduled_tokens.keys())
|
||||
model_runner_output = ModelRunnerOutput(
|
||||
req_ids=scheduled_req_ids,
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(scheduled_req_ids)},
|
||||
sampled_token_ids=[[1]] * len(scheduled_req_ids),
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
)
|
||||
engine_core_outputs = scheduler.update_from_output(output, model_runner_output)
|
||||
assert engine_core_outputs and len(engine_core_outputs) > 0
|
||||
stats = engine_core_outputs[0].scheduler_stats
|
||||
assert stats is not None
|
||||
|
||||
# Verify stats match queue lengths after scheduling
|
||||
assert stats.num_running_reqs == 2 # 2 were scheduled
|
||||
assert stats.num_waiting_reqs == 1 # 1 waiting on capacity
|
||||
assert stats.num_skipped_waiting_reqs == 2 # 2 blocked by constraints
|
||||
|
||||
|
||||
def _assert_right_scheduler_output(
|
||||
output: SchedulerOutput,
|
||||
num_requests: int,
|
||||
|
||||
Reference in New Issue
Block a user