[Core][Metrics] expose waiting request breakdown via labeled metric (capacity/deferred) (#38435)

Signed-off-by: Mukesh Baphna <mukesh@hippocraticai.com>
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
mukesh-hai
2026-04-13 15:30:55 -04:00
committed by GitHub
co-authored by Claude Sonnet 4.6 Mark McLoughlin
parent f72b20976c
commit 5c18b961d6
5 changed files with 101 additions and 4 deletions
@@ -182,6 +182,7 @@ async def test_metrics_counts(
EXPECTED_METRICS_V1 = [
"vllm:num_requests_running",
"vllm:num_requests_waiting",
"vllm:num_requests_waiting_by_reason",
"vllm:kv_cache_usage_perc",
"vllm:prefix_cache_queries",
"vllm:prefix_cache_hits",
+48
View File
@@ -1039,6 +1039,54 @@ def test_no_spec_tokens_scheduled_for_prefill_chunks():
assert len(output.scheduled_spec_decode_tokens[req.request_id]) == num_spec_tokens
def test_scheduler_stats_waiting_queues():
"""Test that scheduler stats correctly report waiting and skipped_waiting queues."""
# Create scheduler with limited capacity so we can have waiting requests
scheduler = create_scheduler(max_num_batched_tokens=100)
# Create requests: some will be scheduled, some will wait on capacity,
# and some will be blocked by constraints
all_requests = create_requests(num_requests=5, num_tokens=50)
# Add 3 requests - only 2 can be scheduled (2 * 50 = 100 tokens)
# The 3rd will remain in waiting queue (capacity constraint)
for request in all_requests[:3]:
scheduler.add_request(request)
# Manually add 2 more to skipped_waiting to simulate constraint-blocked
for request in all_requests[3:]:
request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
scheduler.skipped_waiting.add_request(request)
# Schedule - this will schedule 2 requests, leaving 1 in waiting
output = scheduler.schedule()
# Verify: 2 scheduled, 1 still waiting on capacity, 2 blocked by constraints
assert len(output.scheduled_new_reqs) == 2
assert len(scheduler.waiting) == 1
assert len(scheduler.skipped_waiting) == 2
# Call update_from_output() to get frontend-facing stat
scheduled_req_ids = list(output.num_scheduled_tokens.keys())
model_runner_output = ModelRunnerOutput(
req_ids=scheduled_req_ids,
req_id_to_index={req_id: i for i, req_id in enumerate(scheduled_req_ids)},
sampled_token_ids=[[1]] * len(scheduled_req_ids),
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
)
engine_core_outputs = scheduler.update_from_output(output, model_runner_output)
assert engine_core_outputs and len(engine_core_outputs) > 0
stats = engine_core_outputs[0].scheduler_stats
assert stats is not None
# Verify stats match queue lengths after scheduling
assert stats.num_running_reqs == 2 # 2 were scheduled
assert stats.num_waiting_reqs == 1 # 1 waiting on capacity
assert stats.num_skipped_waiting_reqs == 2 # 2 blocked by constraints
def _assert_right_scheduler_output(
output: SchedulerOutput,
num_requests: int,