[Metrics][P/D] Add KV-fetch stage gauges for async KV loads (#58874)

Signed-off-by: Zijing Liu <liuzijing2014@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Zijing Liu
2026-10-02 01:24:33 +00:00
committed by GitHub
co-authored by Nick Hill Claude Opus 5.5
parent eec3c36a47
commit d58da79de7
5 changed files with 116 additions and 0 deletions
@@ -5,6 +5,7 @@ from unittest.mock import patch
import pytest
from vllm.v1.metrics.stats import KV_FETCH_STAGES
from vllm.v1.outputs import (
EMPTY_MODEL_RUNNER_OUTPUT,
KVConnectorOutput,
@@ -28,6 +29,11 @@ def _num_waiting_requests(scheduler) -> int:
return len(scheduler.waiting) + len(scheduler.kv_holding_waiting)
def _kv_fetch_stages(scheduler) -> tuple[int, ...]:
"""(waiting_to_start, in_progress, completed_waiting) request counts."""
return tuple(scheduler._kv_fetch_counts[s] for s in KV_FETCH_STAGES)
def test_basic_lifecycle():
"""Test lifecycle of a remote prefill."""
vllm_config = create_vllm_config()
@@ -50,10 +56,12 @@ def test_basic_lifecycle():
scheduler.add_request(request)
request_id = request.request_id
assert _kv_fetch_stages(scheduler) == (1, 0, 0)
# STEP (1):
# (1a): schedule()
scheduler_output = scheduler.schedule()
assert _kv_fetch_stages(scheduler) == (0, 1, 0)
# Nothing running and empty scheduler output.
assert len(scheduler.running) == 0
@@ -105,11 +113,13 @@ def test_basic_lifecycle():
)
assert _num_waiting_requests(scheduler) == 1
assert request_id in scheduler.finished_recving_kv_req_ids
assert _kv_fetch_stages(scheduler) == (0, 0, 1)
# STEP (3):
# (3a): schedule(): this should actually schedule.
scheduler_output = scheduler.schedule()
assert len(scheduler.running) == 1
assert _kv_fetch_stages(scheduler) == (0, 0, 0)
# Confirm the block are actually allocated.
num_hashed_blocks = 0
@@ -582,6 +592,24 @@ def test_cannot_recv():
assert_scheduler_empty(scheduler)
def test_kv_fetch_stages_cleared_on_abort():
vllm_config = create_vllm_config()
scheduler = create_scheduler(vllm_config)
BLOCK_SIZE = vllm_config.cache_config.block_size
request = create_request(
request_id=1,
block_size=BLOCK_SIZE,
num_tokens=int(BLOCK_SIZE * 2.5),
do_remote_prefill=True,
)
scheduler.add_request(request)
scheduler.schedule()
assert _kv_fetch_stages(scheduler) == (0, 1, 0)
scheduler.finish_requests(request.request_id, RequestStatus.FINISHED_ABORTED)
assert _kv_fetch_stages(scheduler) == (0, 0, 0)
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler.current_platform"
)
+2
View File
@@ -60,6 +60,8 @@ def assert_scheduler_empty(scheduler: Scheduler):
assert len(scheduler.finished_req_ids) == 0
assert len(scheduler.finished_recving_kv_req_ids) == 0
assert len(scheduler._inflight_prefills) == 0
assert not scheduler._kv_fetch_stages
assert not any(scheduler._kv_fetch_counts.values())
# EncoderCacheManager.
assert len(scheduler.encoder_cache_manager.freed) == 0