[Bugfix][Metrics] Do not log a 0.0% prefix cache hit rate before any query (#54990)
Signed-off-by: Michele Campi <215741962+MicheleCampi@users.noreply.github.com>
This commit is contained in:
@@ -1068,6 +1068,22 @@ def _stats(requests: int, queries: int, hits: int) -> PrefixCacheStats:
|
||||
return PrefixCacheStats(requests=requests, queries=queries, hits=hits)
|
||||
|
||||
|
||||
def test_metrics_empty_distinguishes_no_queries_from_no_hits():
|
||||
"""`hit_rate` alone cannot tell the two apart; `empty` can.
|
||||
|
||||
Both an unobserved window and a genuine all-miss window report a hit
|
||||
rate of 0.0, so anything surfacing that number to a human has to check
|
||||
`empty` first - which is what the prefix-cache log line does.
|
||||
"""
|
||||
metrics = CachingMetrics(max_recent_requests=5)
|
||||
assert metrics.empty
|
||||
assert metrics.hit_rate == 0.0
|
||||
|
||||
metrics.observe(_stats(1, 20, 0))
|
||||
assert not metrics.empty
|
||||
assert metrics.hit_rate == 0.0
|
||||
|
||||
|
||||
def test_metrics():
|
||||
"""Test the prefix caching metrics."""
|
||||
metrics = CachingMetrics(max_recent_requests=5)
|
||||
|
||||
@@ -296,19 +296,16 @@ class LoggingStatLogger(StatLoggerBase):
|
||||
log_parts.append("Preemptions: %d")
|
||||
log_args.append(self.num_preemptions)
|
||||
|
||||
log_parts.extend(
|
||||
[
|
||||
"%s KV cache usage: %.1f%%",
|
||||
"Prefix cache hit rate: %.1f%%",
|
||||
]
|
||||
)
|
||||
log_parts.append("%s KV cache usage: %.1f%%")
|
||||
log_args.extend(
|
||||
[
|
||||
self.kv_cache_device,
|
||||
self.last_scheduler_stats.kv_cache_usage * 100,
|
||||
self.prefix_caching_metrics.hit_rate * 100,
|
||||
]
|
||||
)
|
||||
if not self.prefix_caching_metrics.empty:
|
||||
log_parts.append("Prefix cache hit rate: %.1f%%")
|
||||
log_args.append(self.prefix_caching_metrics.hit_rate * 100)
|
||||
|
||||
if envs.VLLM_COMPUTE_NANS_IN_LOGITS:
|
||||
log_parts.append("Corrupted: %d reqs")
|
||||
|
||||
Reference in New Issue
Block a user