[KV Connector][Metrics] Do not count local prefix cache hits in connector queries (#30522)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
This commit is contained in:
@@ -586,6 +586,7 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
num_external_computed_tokens = 0
|
||||
load_kv_async = False
|
||||
connector_prefix_cache_queries, connector_prefix_cache_hits = 0, 0
|
||||
|
||||
# Get already-cached tokens.
|
||||
if request.num_computed_tokens == 0:
|
||||
@@ -613,6 +614,11 @@ class Scheduler(SchedulerInterface):
|
||||
request.num_external_computed_tokens = ext_tokens
|
||||
num_external_computed_tokens = ext_tokens
|
||||
|
||||
connector_prefix_cache_queries = (
|
||||
request.num_tokens - num_new_local_computed_tokens
|
||||
)
|
||||
connector_prefix_cache_hits = num_external_computed_tokens
|
||||
|
||||
# Total computed tokens (local + external).
|
||||
num_computed_tokens = (
|
||||
num_new_local_computed_tokens + num_external_computed_tokens
|
||||
@@ -728,6 +734,15 @@ class Scheduler(SchedulerInterface):
|
||||
self.kv_cache_manager.get_blocks(request_id),
|
||||
num_external_computed_tokens,
|
||||
)
|
||||
if (
|
||||
self.connector_prefix_cache_stats is not None
|
||||
and connector_prefix_cache_queries != 0
|
||||
):
|
||||
self.connector_prefix_cache_stats.record(
|
||||
num_tokens=connector_prefix_cache_queries,
|
||||
num_hits=connector_prefix_cache_hits,
|
||||
preempted=request.num_preemptions > 0,
|
||||
)
|
||||
|
||||
# Request was already popped from self.waiting
|
||||
# unless it was re-added above due to new_blocks being None.
|
||||
@@ -739,8 +754,6 @@ class Scheduler(SchedulerInterface):
|
||||
request.status = RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
continue
|
||||
|
||||
self._update_connector_prefix_cache_stats(request)
|
||||
|
||||
self.running.append(request)
|
||||
if self.log_stats:
|
||||
request.record_event(
|
||||
@@ -1805,7 +1818,10 @@ class Scheduler(SchedulerInterface):
|
||||
return None
|
||||
prefix_cache_stats = self.kv_cache_manager.make_prefix_cache_stats()
|
||||
assert prefix_cache_stats is not None
|
||||
connector_prefix_cache_stats = self._make_connector_prefix_cache_stats()
|
||||
connector_prefix_cache_stats: PrefixCacheStats | None = None
|
||||
if self.connector_prefix_cache_stats is not None:
|
||||
connector_prefix_cache_stats = self.connector_prefix_cache_stats
|
||||
self.connector_prefix_cache_stats = PrefixCacheStats()
|
||||
eviction_events = (
|
||||
self.kv_metrics_collector.drain_events()
|
||||
if self.kv_metrics_collector is not None
|
||||
@@ -1866,23 +1882,6 @@ class Scheduler(SchedulerInterface):
|
||||
# KV Connector Related Methods
|
||||
########################################################################
|
||||
|
||||
def _update_connector_prefix_cache_stats(self, request: Request) -> None:
|
||||
if self.connector_prefix_cache_stats is None:
|
||||
return
|
||||
|
||||
self.connector_prefix_cache_stats.record(
|
||||
num_tokens=request.num_tokens,
|
||||
num_hits=request.num_external_computed_tokens,
|
||||
preempted=request.num_preemptions > 0,
|
||||
)
|
||||
|
||||
def _make_connector_prefix_cache_stats(self) -> PrefixCacheStats | None:
|
||||
if self.connector_prefix_cache_stats is None:
|
||||
return None
|
||||
stats = self.connector_prefix_cache_stats
|
||||
self.connector_prefix_cache_stats = PrefixCacheStats()
|
||||
return stats
|
||||
|
||||
def get_kv_connector(self) -> KVConnectorBase_V1 | None:
|
||||
return self.connector
|
||||
|
||||
|
||||
Reference in New Issue
Block a user