[KV-Offloading] : Expose CPU cache usage metric (#45737)
Signed-off-by: Varun Sundar Rabindranath <varun-sundar-rabindranath@h100-01.nemg-001.lab.rdu2.dc.redhat.com> Signed-off-by: <> Co-authored-by: Varun Sundar Rabindranath <varun-sundar-rabindranath@h100-01.nemg-001.lab.rdu2.dc.redhat.com>
This commit is contained in:
co-authored by
Varun Sundar Rabindranath
parent
cc22621b51
commit
3b4a76b63f
@@ -14,11 +14,15 @@ from vllm.v1.kv_offload.base import (
|
||||
GPULoadStoreSpec,
|
||||
LoadStoreSpec,
|
||||
OffloadingCounterMetadata,
|
||||
OffloadingGaugeMetadata,
|
||||
OffloadingManager,
|
||||
OffloadingMetricMetadata,
|
||||
OffloadingSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.common import METRIC_STORES_SKIPPED, CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.common import (
|
||||
CPULoadStoreSpec,
|
||||
CPUOffloadingMetrics,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.worker.worker import OffloadingHandler
|
||||
@@ -31,17 +35,27 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
def build_metric_definitions(
|
||||
cls, extra_config: dict[str, Any]
|
||||
) -> dict[str, OffloadingMetricMetadata]:
|
||||
store_threshold = int(extra_config.get("store_threshold", 0))
|
||||
if store_threshold < 2:
|
||||
return {}
|
||||
return {
|
||||
METRIC_STORES_SKIPPED: OffloadingCounterMetadata(
|
||||
definitions: dict[str, OffloadingMetricMetadata] = {
|
||||
CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC: OffloadingGaugeMetadata(
|
||||
documentation=(
|
||||
"Number of KV offload stores skipped because the reuse "
|
||||
"threshold was not reached."
|
||||
"Fraction of CPU KV-cache space currently pinned by active "
|
||||
"transfers (0.0 = idle, 1.0 = saturated). Sustained high "
|
||||
"values indicate transfers (stores or promotions) may be "
|
||||
"dropped due to insufficient capacity."
|
||||
),
|
||||
)
|
||||
}
|
||||
store_threshold = int(extra_config.get("store_threshold", 0))
|
||||
if store_threshold >= 2:
|
||||
definitions[CPUOffloadingMetrics.STORES_SKIPPED] = (
|
||||
OffloadingCounterMetadata(
|
||||
documentation=(
|
||||
"Number of KV offload stores skipped because the reuse "
|
||||
"threshold was not reached."
|
||||
),
|
||||
)
|
||||
)
|
||||
return definitions
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig):
|
||||
super().__init__(vllm_config, kv_cache_config)
|
||||
|
||||
Reference in New Issue
Block a user