[KV-Offloading] : Expose CPU cache usage metric (#45737)
Signed-off-by: Varun Sundar Rabindranath <varun-sundar-rabindranath@h100-01.nemg-001.lab.rdu2.dc.redhat.com> Signed-off-by: <> Co-authored-by: Varun Sundar Rabindranath <varun-sundar-rabindranath@h100-01.nemg-001.lab.rdu2.dc.redhat.com>
This commit is contained in:
co-authored by
Varun Sundar Rabindranath
parent
cc22621b51
commit
3b4a76b63f
@@ -14,12 +14,13 @@ from vllm.v1.kv_offload.base import (
|
||||
ReqContext,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
||||
from vllm.v1.kv_offload.cpu.common import (
|
||||
CPULoadStoreSpec,
|
||||
CPUOffloadingMetrics,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
|
||||
|
||||
STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
|
||||
|
||||
|
||||
def make_req_context(
|
||||
req_id: str = "", kv_transfer_params: dict | None = None
|
||||
@@ -181,10 +182,45 @@ def test_filter_reused_manager_reports_stores_skipped_counter():
|
||||
)
|
||||
stats = manager.get_stats()
|
||||
assert stats is not None
|
||||
assert stats.reduce()[STORES_SKIPPED] == 3
|
||||
assert stats.reduce()[CPUOffloadingMetrics.STORES_SKIPPED] == 3
|
||||
stats = manager.get_stats()
|
||||
assert stats is not None
|
||||
assert stats.reduce()[STORES_SKIPPED] == 0
|
||||
assert stats.reduce()[CPUOffloadingMetrics.STORES_SKIPPED] == 0
|
||||
|
||||
|
||||
def test_cpu_manager_reports_cache_usage_gauge():
|
||||
def check_usage_stats(manager: CPUOffloadingManager, value: float):
|
||||
stats = manager.get_stats()
|
||||
assert stats is not None
|
||||
assert stats.reduce()[
|
||||
CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC
|
||||
] == pytest.approx(value)
|
||||
|
||||
# Zero-capacity manager always reports 0.0
|
||||
manager = make_cpu_manager(num_blocks=0)
|
||||
check_usage_stats(manager, 0.0)
|
||||
|
||||
# Empty manager (4 blocks, none allocated): usage = 0.0
|
||||
manager = make_cpu_manager(num_blocks=4)
|
||||
check_usage_stats(manager, 0.0)
|
||||
|
||||
# After allocating 2 of 4 blocks: usage = 0.5
|
||||
manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
check_usage_stats(manager, 0.5)
|
||||
|
||||
# After filling all 4 blocks: usage = 1.0
|
||||
manager.prepare_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
||||
check_usage_stats(manager, 1.0)
|
||||
|
||||
# After completing store, the blocks becomes evictable as it is not actively used
|
||||
# and usage drops.
|
||||
manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
|
||||
check_usage_stats(manager, 0.5)
|
||||
|
||||
# After completing store, the blocks becomes evictable as it is not actively used
|
||||
# and usage drops.
|
||||
manager.complete_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
|
||||
check_usage_stats(manager, 0.0)
|
||||
|
||||
|
||||
def test_cpu_manager():
|
||||
|
||||
Reference in New Issue
Block a user