[Metrics] Expose cached prompt tokens by cache tier (#56318)

Signed-off-by: Cam Quilici <cjquilici@gmail.com>
Signed-off-by: Cam Quilici <cameron@semianalysis.com>
Co-authored-by: Cam Quilici <cameron@semianalysis.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Cameron Quilici
2026-10-06 14:52:05 -04:00
committed by GitHub
co-authored by Cam Quilici Nick Hill Yifan Qiao
parent ad0f67a7ec
commit e37e51dd24
36 changed files with 1005 additions and 44 deletions
+11 -1
View File
@@ -1261,10 +1261,20 @@ class Scheduler(SchedulerInterface):
self.connector_prefix_cache_stats is not None
and connector_prefix_cache_queries != 0
):
preempted = request.num_preemptions > 0
self.connector_prefix_cache_stats.record(
num_tokens=connector_prefix_cache_queries,
num_hits=connector_prefix_cache_hits,
preempted=request.num_preemptions > 0,
preempted=preempted,
# The connector knows the tier only after the load
# plan above exists.
hits_by_source=(
self.connector.get_external_cache_hit_sources(
request, connector_prefix_cache_hits
)
if connector_prefix_cache_hits and not preempted
else None
),
)
# Record at admission so unscheduled lookups are not counted.