[Metrics] Expose cached prompt tokens by cache tier (#56318)

Signed-off-by: Cam Quilici <cjquilici@gmail.com>
Signed-off-by: Cam Quilici <cameron@semianalysis.com>
Co-authored-by: Cam Quilici <cameron@semianalysis.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Cameron Quilici
2026-10-06 14:52:05 -04:00
committed by GitHub
co-authored by Cam Quilici Nick Hill Yifan Qiao
parent ad0f67a7ec
commit e37e51dd24
36 changed files with 1005 additions and 44 deletions
+18 -4
View File
@@ -17,6 +17,7 @@ import pytest
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
from vllm.v1.metrics.cache_hit_source import CacheHitSource
# ---------------------------------------------------------------------------
# Fixtures
@@ -43,12 +44,22 @@ def _make_mock_args():
def test_pre_registered_tiers_can_be_imported():
"""CI sentinel: registered tiers import and yield SecondaryTierManager."""
# Derived from each tier's ``medium``; P2P has none and sets its own.
expected_sources = {
"example": CacheHitSource.HOST,
"fs": CacheHitSource.DISK,
"obj": CacheHitSource.DISK,
"p2p": CacheHitSource.P2P,
}
for tier_type in SecondaryTierFactory._registry:
# KVCR is an optional external dependency and may not be installed.
if tier_type == "kvcr" and find_spec("kvcr") is None:
continue
cls = SecondaryTierFactory._registry[tier_type]()
assert issubclass(cls, SecondaryTierManager)
assert cls.cache_hit_source == expected_sources.get(
tier_type, CacheHitSource.EXTERNAL_UNSPECIFIED
)
def test_example_tier_registered():
@@ -95,7 +106,8 @@ def test_create_multiple_tiers():
assert all(isinstance(tier, ExampleSecondaryTierManager) for tier in tiers)
def test_register_new_tier_type():
@pytest.mark.parametrize("tier_type", ["custom_tier", "cpu", "nvme"])
def test_register_new_tier_type(tier_type):
"""Verify that new tier types can be registered and created.
This is how external projects add custom secondary tiers
@@ -103,20 +115,22 @@ def test_register_new_tier_type():
"""
# Register a new tier type (reuse example manager for simplicity)
SecondaryTierFactory.register_tier(
"custom_tier",
tier_type,
"vllm.v1.kv_offload.tiering.example.manager",
"ExampleSecondaryTierManager",
)
primary_kv_view, offloading_spec = _make_mock_args()
tier = SecondaryTierFactory.create_secondary_tier(
{"type": "custom_tier", "custom_param": 99},
{"type": tier_type, "custom_param": 99},
primary_kv_view,
offloading_spec,
)
assert tier.tier_type == "custom_tier"
assert tier.tier_type == tier_type
assert isinstance(tier, ExampleSecondaryTierManager)
# Attribution follows the tier's medium, never its registration name.
assert tier.cache_hit_source is CacheHitSource.HOST
# ---------------------------------------------------------------------------
@@ -53,6 +53,7 @@ from vllm.v1.kv_offload.tiering.manager import (
TieringOffloadingManager,
)
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
from vllm.v1.metrics.cache_hit_source import CacheHitSource
_CTX = ReqContext(req_id="test")
_MOCK_OFFLOADING_SPEC = MagicMock()
@@ -500,8 +501,13 @@ class TestTieringOffloadingManager:
# Lookup should find all chunks in primary
assert count_hits(self.manager, chunks) == 3
def test_promotion_from_secondary(self, manager_setup):
@pytest.mark.parametrize(
"source",
[CacheHitSource.EXTERNAL_UNSPECIFIED, CacheHitSource.DISK, CacheHitSource.P2P],
)
def test_promotion_from_secondary(self, manager_setup, monkeypatch, source):
"""Test promotion of chunks from secondary to primary tier."""
monkeypatch.setattr(ExampleSecondaryTierManager, "cache_hit_source", source)
chunks = to_keys(range(3))
# Manually add chunks to secondary tier (simulate previous cascade)
@@ -525,6 +531,27 @@ class TestTieringOffloadingManager:
# Next lookup should succeed
assert count_hits(self.manager, chunks) == 3
# The request that caused the promotion retains the secondary origin.
assert all(
self.manager.get_load_source(chunk, _CTX) == source for chunk in chunks
)
# A later request sees the chunks as ordinary host-memory primary hits.
later_context = ReqContext(req_id="later")
self.manager.on_new_request(later_context)
assert all(
self.manager.lookup(chunk, later_context) is LookupResult.HIT
for chunk in chunks
)
assert all(
self.manager.get_load_source(chunk, later_context) == "host"
for chunk in chunks
)
@pytest.mark.parametrize(
"source",
[CacheHitSource.EXTERNAL_UNSPECIFIED, CacheHitSource.DISK, CacheHitSource.P2P],
)
@pytest.mark.parametrize(
("successful_indices", "expected_results"),
[
@@ -540,8 +567,9 @@ class TestTieringOffloadingManager:
ids=["partial", "legacy-full-failure"],
)
def test_failed_promotion_keeps_only_successful_chunks(
self, manager_setup, successful_indices, expected_results
self, manager_setup, monkeypatch, source, successful_indices, expected_results
):
monkeypatch.setattr(ExampleSecondaryTierManager, "cache_hit_source", source)
chunks = to_keys(range(3))
for chunk in chunks:
self.secondary_tier1.chunks[chunk] = True
@@ -571,6 +599,14 @@ class TestTieringOffloadingManager:
assert [
self.primary_tier.lookup(chunk, _CTX) for chunk in chunks
] == expected_results
expected_source_keys = (
set()
if successful_indices is None
else {chunks[i] for i in successful_indices}
)
assert set(self.manager._request_load_sources[_CTX.req_id]) == (
expected_source_keys
)
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
"""Resolved lookups report one sync delay sample per tier and chunk."""
@@ -1315,6 +1351,7 @@ class TestTieringOffloadingManager:
# Orchestrator state cleared.
assert self.manager._jobs == {}
assert self.manager._pending_load_submissions == {}
assert self.manager._request_load_sources == {}
assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id}
assert self.manager._processed_jobs_this_step is False