[Bugfix][Metrics] Do not log a 0.0% prefix cache hit rate before any query (#54990)

Signed-off-by: Michele Campi <215741962+MicheleCampi@users.noreply.github.com>
This commit is contained in:
Michele Campi
2026-09-17 18:19:13 +00:00
committed by GitHub
parent 09c379ab61
commit 6a344d8c85
2 changed files with 20 additions and 7 deletions
+16
View File
@@ -1068,6 +1068,22 @@ def _stats(requests: int, queries: int, hits: int) -> PrefixCacheStats:
return PrefixCacheStats(requests=requests, queries=queries, hits=hits)
def test_metrics_empty_distinguishes_no_queries_from_no_hits():
"""`hit_rate` alone cannot tell the two apart; `empty` can.
Both an unobserved window and a genuine all-miss window report a hit
rate of 0.0, so anything surfacing that number to a human has to check
`empty` first - which is what the prefix-cache log line does.
"""
metrics = CachingMetrics(max_recent_requests=5)
assert metrics.empty
assert metrics.hit_rate == 0.0
metrics.observe(_stats(1, 20, 0))
assert not metrics.empty
assert metrics.hit_rate == 0.0
def test_metrics():
"""Test the prefix caching metrics."""
metrics = CachingMetrics(max_recent_requests=5)
+4 -7
View File
@@ -296,19 +296,16 @@ class LoggingStatLogger(StatLoggerBase):
log_parts.append("Preemptions: %d")
log_args.append(self.num_preemptions)
log_parts.extend(
[
"%s KV cache usage: %.1f%%",
"Prefix cache hit rate: %.1f%%",
]
)
log_parts.append("%s KV cache usage: %.1f%%")
log_args.extend(
[
self.kv_cache_device,
self.last_scheduler_stats.kv_cache_usage * 100,
self.prefix_caching_metrics.hit_rate * 100,
]
)
if not self.prefix_caching_metrics.empty:
log_parts.append("Prefix cache hit rate: %.1f%%")
log_args.append(self.prefix_caching_metrics.hit_rate * 100)
if envs.VLLM_COMPUTE_NANS_IN_LOGITS:
log_parts.append("Corrupted: %d reqs")