[Metrics] Expose cached prompt tokens by cache tier (#56318)
Signed-off-by: Cam Quilici <cjquilici@gmail.com> Signed-off-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Cam Quilici
Nick Hill
Yifan Qiao
parent
ad0f67a7ec
commit
e37e51dd24
@@ -17,6 +17,7 @@ import pytest
|
||||
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
|
||||
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
|
||||
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
@@ -43,12 +44,22 @@ def _make_mock_args():
|
||||
|
||||
def test_pre_registered_tiers_can_be_imported():
|
||||
"""CI sentinel: registered tiers import and yield SecondaryTierManager."""
|
||||
# Derived from each tier's ``medium``; P2P has none and sets its own.
|
||||
expected_sources = {
|
||||
"example": CacheHitSource.HOST,
|
||||
"fs": CacheHitSource.DISK,
|
||||
"obj": CacheHitSource.DISK,
|
||||
"p2p": CacheHitSource.P2P,
|
||||
}
|
||||
for tier_type in SecondaryTierFactory._registry:
|
||||
# KVCR is an optional external dependency and may not be installed.
|
||||
if tier_type == "kvcr" and find_spec("kvcr") is None:
|
||||
continue
|
||||
cls = SecondaryTierFactory._registry[tier_type]()
|
||||
assert issubclass(cls, SecondaryTierManager)
|
||||
assert cls.cache_hit_source == expected_sources.get(
|
||||
tier_type, CacheHitSource.EXTERNAL_UNSPECIFIED
|
||||
)
|
||||
|
||||
|
||||
def test_example_tier_registered():
|
||||
@@ -95,7 +106,8 @@ def test_create_multiple_tiers():
|
||||
assert all(isinstance(tier, ExampleSecondaryTierManager) for tier in tiers)
|
||||
|
||||
|
||||
def test_register_new_tier_type():
|
||||
@pytest.mark.parametrize("tier_type", ["custom_tier", "cpu", "nvme"])
|
||||
def test_register_new_tier_type(tier_type):
|
||||
"""Verify that new tier types can be registered and created.
|
||||
|
||||
This is how external projects add custom secondary tiers
|
||||
@@ -103,20 +115,22 @@ def test_register_new_tier_type():
|
||||
"""
|
||||
# Register a new tier type (reuse example manager for simplicity)
|
||||
SecondaryTierFactory.register_tier(
|
||||
"custom_tier",
|
||||
tier_type,
|
||||
"vllm.v1.kv_offload.tiering.example.manager",
|
||||
"ExampleSecondaryTierManager",
|
||||
)
|
||||
|
||||
primary_kv_view, offloading_spec = _make_mock_args()
|
||||
tier = SecondaryTierFactory.create_secondary_tier(
|
||||
{"type": "custom_tier", "custom_param": 99},
|
||||
{"type": tier_type, "custom_param": 99},
|
||||
primary_kv_view,
|
||||
offloading_spec,
|
||||
)
|
||||
|
||||
assert tier.tier_type == "custom_tier"
|
||||
assert tier.tier_type == tier_type
|
||||
assert isinstance(tier, ExampleSecondaryTierManager)
|
||||
# Attribution follows the tier's medium, never its registration name.
|
||||
assert tier.cache_hit_source is CacheHitSource.HOST
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -53,6 +53,7 @@ from vllm.v1.kv_offload.tiering.manager import (
|
||||
TieringOffloadingManager,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
|
||||
_CTX = ReqContext(req_id="test")
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock()
|
||||
@@ -500,8 +501,13 @@ class TestTieringOffloadingManager:
|
||||
# Lookup should find all chunks in primary
|
||||
assert count_hits(self.manager, chunks) == 3
|
||||
|
||||
def test_promotion_from_secondary(self, manager_setup):
|
||||
@pytest.mark.parametrize(
|
||||
"source",
|
||||
[CacheHitSource.EXTERNAL_UNSPECIFIED, CacheHitSource.DISK, CacheHitSource.P2P],
|
||||
)
|
||||
def test_promotion_from_secondary(self, manager_setup, monkeypatch, source):
|
||||
"""Test promotion of chunks from secondary to primary tier."""
|
||||
monkeypatch.setattr(ExampleSecondaryTierManager, "cache_hit_source", source)
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Manually add chunks to secondary tier (simulate previous cascade)
|
||||
@@ -525,6 +531,27 @@ class TestTieringOffloadingManager:
|
||||
# Next lookup should succeed
|
||||
assert count_hits(self.manager, chunks) == 3
|
||||
|
||||
# The request that caused the promotion retains the secondary origin.
|
||||
assert all(
|
||||
self.manager.get_load_source(chunk, _CTX) == source for chunk in chunks
|
||||
)
|
||||
|
||||
# A later request sees the chunks as ordinary host-memory primary hits.
|
||||
later_context = ReqContext(req_id="later")
|
||||
self.manager.on_new_request(later_context)
|
||||
assert all(
|
||||
self.manager.lookup(chunk, later_context) is LookupResult.HIT
|
||||
for chunk in chunks
|
||||
)
|
||||
assert all(
|
||||
self.manager.get_load_source(chunk, later_context) == "host"
|
||||
for chunk in chunks
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source",
|
||||
[CacheHitSource.EXTERNAL_UNSPECIFIED, CacheHitSource.DISK, CacheHitSource.P2P],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("successful_indices", "expected_results"),
|
||||
[
|
||||
@@ -540,8 +567,9 @@ class TestTieringOffloadingManager:
|
||||
ids=["partial", "legacy-full-failure"],
|
||||
)
|
||||
def test_failed_promotion_keeps_only_successful_chunks(
|
||||
self, manager_setup, successful_indices, expected_results
|
||||
self, manager_setup, monkeypatch, source, successful_indices, expected_results
|
||||
):
|
||||
monkeypatch.setattr(ExampleSecondaryTierManager, "cache_hit_source", source)
|
||||
chunks = to_keys(range(3))
|
||||
for chunk in chunks:
|
||||
self.secondary_tier1.chunks[chunk] = True
|
||||
@@ -571,6 +599,14 @@ class TestTieringOffloadingManager:
|
||||
assert [
|
||||
self.primary_tier.lookup(chunk, _CTX) for chunk in chunks
|
||||
] == expected_results
|
||||
expected_source_keys = (
|
||||
set()
|
||||
if successful_indices is None
|
||||
else {chunks[i] for i in successful_indices}
|
||||
)
|
||||
assert set(self.manager._request_load_sources[_CTX.req_id]) == (
|
||||
expected_source_keys
|
||||
)
|
||||
|
||||
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
|
||||
"""Resolved lookups report one sync delay sample per tier and chunk."""
|
||||
@@ -1315,6 +1351,7 @@ class TestTieringOffloadingManager:
|
||||
# Orchestrator state cleared.
|
||||
assert self.manager._jobs == {}
|
||||
assert self.manager._pending_load_submissions == {}
|
||||
assert self.manager._request_load_sources == {}
|
||||
assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id}
|
||||
assert self.manager._processed_jobs_this_step is False
|
||||
|
||||
|
||||
Reference in New Issue
Block a user