[Metrics][P/D] Add KV-fetch stage gauges for async KV loads (#58874)
Signed-off-by: Zijing Liu <liuzijing2014@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Nick Hill
Claude Opus 5.5
parent
eec3c36a47
commit
d58da79de7
@@ -5,6 +5,7 @@ from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.v1.metrics.stats import KV_FETCH_STAGES
|
||||
from vllm.v1.outputs import (
|
||||
EMPTY_MODEL_RUNNER_OUTPUT,
|
||||
KVConnectorOutput,
|
||||
@@ -28,6 +29,11 @@ def _num_waiting_requests(scheduler) -> int:
|
||||
return len(scheduler.waiting) + len(scheduler.kv_holding_waiting)
|
||||
|
||||
|
||||
def _kv_fetch_stages(scheduler) -> tuple[int, ...]:
|
||||
"""(waiting_to_start, in_progress, completed_waiting) request counts."""
|
||||
return tuple(scheduler._kv_fetch_counts[s] for s in KV_FETCH_STAGES)
|
||||
|
||||
|
||||
def test_basic_lifecycle():
|
||||
"""Test lifecycle of a remote prefill."""
|
||||
vllm_config = create_vllm_config()
|
||||
@@ -50,10 +56,12 @@ def test_basic_lifecycle():
|
||||
|
||||
scheduler.add_request(request)
|
||||
request_id = request.request_id
|
||||
assert _kv_fetch_stages(scheduler) == (1, 0, 0)
|
||||
|
||||
# STEP (1):
|
||||
# (1a): schedule()
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert _kv_fetch_stages(scheduler) == (0, 1, 0)
|
||||
|
||||
# Nothing running and empty scheduler output.
|
||||
assert len(scheduler.running) == 0
|
||||
@@ -105,11 +113,13 @@ def test_basic_lifecycle():
|
||||
)
|
||||
assert _num_waiting_requests(scheduler) == 1
|
||||
assert request_id in scheduler.finished_recving_kv_req_ids
|
||||
assert _kv_fetch_stages(scheduler) == (0, 0, 1)
|
||||
|
||||
# STEP (3):
|
||||
# (3a): schedule(): this should actually schedule.
|
||||
scheduler_output = scheduler.schedule()
|
||||
assert len(scheduler.running) == 1
|
||||
assert _kv_fetch_stages(scheduler) == (0, 0, 0)
|
||||
|
||||
# Confirm the block are actually allocated.
|
||||
num_hashed_blocks = 0
|
||||
@@ -582,6 +592,24 @@ def test_cannot_recv():
|
||||
assert_scheduler_empty(scheduler)
|
||||
|
||||
|
||||
def test_kv_fetch_stages_cleared_on_abort():
|
||||
vllm_config = create_vllm_config()
|
||||
scheduler = create_scheduler(vllm_config)
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
request = create_request(
|
||||
request_id=1,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=int(BLOCK_SIZE * 2.5),
|
||||
do_remote_prefill=True,
|
||||
)
|
||||
scheduler.add_request(request)
|
||||
scheduler.schedule()
|
||||
assert _kv_fetch_stages(scheduler) == (0, 1, 0)
|
||||
|
||||
scheduler.finish_requests(request.request_id, RequestStatus.FINISHED_ABORTED)
|
||||
assert _kv_fetch_stages(scheduler) == (0, 0, 0)
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler.current_platform"
|
||||
)
|
||||
|
||||
@@ -60,6 +60,8 @@ def assert_scheduler_empty(scheduler: Scheduler):
|
||||
assert len(scheduler.finished_req_ids) == 0
|
||||
assert len(scheduler.finished_recving_kv_req_ids) == 0
|
||||
assert len(scheduler._inflight_prefills) == 0
|
||||
assert not scheduler._kv_fetch_stages
|
||||
assert not any(scheduler._kv_fetch_counts.values())
|
||||
|
||||
# EncoderCacheManager.
|
||||
assert len(scheduler.encoder_cache_manager.freed) == 0
|
||||
|
||||
Reference in New Issue
Block a user