[Metrics][KV Offload] Add Prometheus metrics for SimpleCPUOffloadConnector (#57251)
Signed-off-by: Vincent <vincexxchan@gmail.com>
This commit is contained in:
@@ -15,6 +15,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
@@ -25,6 +31,10 @@ from vllm.v1.simple_kv_offload.manager import (
|
||||
from vllm.v1.simple_kv_offload.metadata import (
|
||||
SimpleCPUOffloadMetadata,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.metrics import (
|
||||
SimpleCPUOffloadPromMetrics,
|
||||
SimpleCPUOffloadStats,
|
||||
)
|
||||
from vllm.v1.simple_kv_offload.worker import (
|
||||
SimpleCPUOffloadWorker,
|
||||
)
|
||||
@@ -162,6 +172,7 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
hash_block_size=hash_block_size,
|
||||
lazy_offload=lazy_offload,
|
||||
disk_capacity_bytes=disk_capacity_bytes if disk_mode else 0,
|
||||
use_page_cache=use_page_cache if disk_mode else False,
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.worker_handler = SimpleCPUOffloadWorker(
|
||||
@@ -307,6 +318,38 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return self.scheduler_manager.get_boundary_store_stats()
|
||||
return None
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
# Worker-side callers may invoke this hook on connectors constructed
|
||||
# via __new__ (as in test_worker.py), which lack scheduler_manager.
|
||||
scheduler_manager: SimpleCPUOffloadScheduler | None = getattr(
|
||||
self, "scheduler_manager", None
|
||||
)
|
||||
if scheduler_manager is not None:
|
||||
return scheduler_manager.get_stats()
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
SimpleCPUOffloadStats(data=data)
|
||||
if data is not None
|
||||
else SimpleCPUOffloadStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return SimpleCPUOffloadPromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
if self.scheduler_manager is not None:
|
||||
return self.scheduler_manager.take_events()
|
||||
|
||||
Reference in New Issue
Block a user