[Metrics][P/D] Add KV-fetch stage gauges for async KV loads (#58874)
Signed-off-by: Zijing Liu <liuzijing2014@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Nick Hill
Claude Opus 5.5
parent
eec3c36a47
commit
d58da79de7
@@ -60,6 +60,8 @@ def assert_scheduler_empty(scheduler: Scheduler):
|
||||
assert len(scheduler.finished_req_ids) == 0
|
||||
assert len(scheduler.finished_recving_kv_req_ids) == 0
|
||||
assert len(scheduler._inflight_prefills) == 0
|
||||
assert not scheduler._kv_fetch_stages
|
||||
assert not any(scheduler._kv_fetch_counts.values())
|
||||
|
||||
# EncoderCacheManager.
|
||||
assert len(scheduler.encoder_cache_manager.freed) == 0
|
||||
|
||||
Reference in New Issue
Block a user