[Metrics] Expose cached prompt tokens by cache tier (#56318)
Signed-off-by: Cam Quilici <cjquilici@gmail.com> Signed-off-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Cam Quilici
Nick Hill
Yifan Qiao
parent
ad0f67a7ec
commit
e37e51dd24
@@ -1261,10 +1261,20 @@ class Scheduler(SchedulerInterface):
|
||||
self.connector_prefix_cache_stats is not None
|
||||
and connector_prefix_cache_queries != 0
|
||||
):
|
||||
preempted = request.num_preemptions > 0
|
||||
self.connector_prefix_cache_stats.record(
|
||||
num_tokens=connector_prefix_cache_queries,
|
||||
num_hits=connector_prefix_cache_hits,
|
||||
preempted=request.num_preemptions > 0,
|
||||
preempted=preempted,
|
||||
# The connector knows the tier only after the load
|
||||
# plan above exists.
|
||||
hits_by_source=(
|
||||
self.connector.get_external_cache_hit_sources(
|
||||
request, connector_prefix_cache_hits
|
||||
)
|
||||
if connector_prefix_cache_hits and not preempted
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
# Record at admission so unscheduled lookups are not counted.
|
||||
|
||||
Reference in New Issue
Block a user