[Metrics][P/D] Add KV-fetch stage gauges for async KV loads (#58874)

Signed-off-by: Zijing Liu <liuzijing2014@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Zijing Liu
2026-10-02 01:24:33 +00:00
committed by GitHub
co-authored by Nick Hill Claude Opus 5.5
parent eec3c36a47
commit d58da79de7
5 changed files with 116 additions and 0 deletions
+2
View File
@@ -60,6 +60,8 @@ def assert_scheduler_empty(scheduler: Scheduler):
assert len(scheduler.finished_req_ids) == 0
assert len(scheduler.finished_recving_kv_req_ids) == 0
assert len(scheduler._inflight_prefills) == 0
assert not scheduler._kv_fetch_stages
assert not any(scheduler._kv_fetch_counts.values())
# EncoderCacheManager.
assert len(scheduler.encoder_cache_manager.freed) == 0