[KV-Offloading] : Expose CPU cache usage metric (#45737)
Signed-off-by: Varun Sundar Rabindranath <varun-sundar-rabindranath@h100-01.nemg-001.lab.rdu2.dc.redhat.com> Signed-off-by: <> Co-authored-by: Varun Sundar Rabindranath <varun-sundar-rabindranath@h100-01.nemg-001.lab.rdu2.dc.redhat.com>
This commit is contained in:
co-authored by
Varun Sundar Rabindranath
parent
cc22621b51
commit
3b4a76b63f
@@ -18,7 +18,10 @@ from vllm.v1.kv_offload.base import (
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.common import METRIC_STORES_SKIPPED, CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.common import (
|
||||
CPULoadStoreSpec,
|
||||
CPUOffloadingMetrics,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
||||
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
|
||||
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
|
||||
@@ -282,13 +285,21 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
self.events.clear()
|
||||
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
if self.store_threshold < 2:
|
||||
return None
|
||||
|
||||
stats = OffloadingConnectorStats()
|
||||
stats.increase_counter(
|
||||
METRIC_STORES_SKIPPED,
|
||||
self.stores_skipped_in_current_batch,
|
||||
|
||||
# Compute cache usage.
|
||||
num_used = (
|
||||
self._num_allocated_blocks
|
||||
- len(self._free_list)
|
||||
- self._num_evictable_cache_blocks
|
||||
)
|
||||
self.stores_skipped_in_current_batch = 0
|
||||
usage = num_used / self._num_blocks if self._num_blocks > 0 else 0.0
|
||||
stats.set_gauge(CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC, usage)
|
||||
|
||||
if self.store_threshold >= 2:
|
||||
stats.increase_counter(
|
||||
CPUOffloadingMetrics.STORES_SKIPPED,
|
||||
self.stores_skipped_in_current_batch,
|
||||
)
|
||||
self.stores_skipped_in_current_batch = 0
|
||||
return stats
|
||||
|
||||
Reference in New Issue
Block a user