[Metrics] Expose cached prompt tokens by cache tier (#56318)

Signed-off-by: Cam Quilici <cjquilici@gmail.com>
Signed-off-by: Cam Quilici <cameron@semianalysis.com>
Co-authored-by: Cam Quilici <cameron@semianalysis.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Cameron Quilici
2026-10-06 14:52:05 -04:00
committed by GitHub
co-authored by Cam Quilici Nick Hill Yifan Qiao
parent ad0f67a7ec
commit e37e51dd24
36 changed files with 1005 additions and 44 deletions
+61 -1
View File
@@ -3,7 +3,7 @@
import dataclasses
from concurrent.futures import Future
from types import SimpleNamespace
from unittest.mock import Mock
from unittest.mock import Mock, patch
import numpy as np
import pytest
@@ -60,6 +60,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheGroupSpec,
MambaSpec,
)
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import (
DraftTokenIds,
ECConnectorOutput,
@@ -2613,6 +2614,65 @@ def test_kv_connector_honors_skip_reading_prefix_cache():
assert output.num_scheduled_tokens[scoring.request_id] == BLOCK_SIZE * 4
@pytest.mark.parametrize("is_async", [False, True])
def test_kv_connector_records_external_cache_hit_sources(monkeypatch, is_async):
block_size = 16
num_matched_tokens = 2 * block_size
scheduler = create_scheduler(
enable_prefix_caching=True,
use_kv_connector=mock_kv(
matched_tokens=num_matched_tokens,
is_async=is_async,
),
block_size=block_size,
)
request = create_requests(
num_requests=1,
num_tokens=2 * num_matched_tokens,
block_size=block_size,
)[0]
assert scheduler.connector is not None
get_sources = Mock(
return_value={CacheHitSource.P2P: block_size, CacheHitSource.HOST: block_size}
)
update_state = Mock(wraps=scheduler.connector.update_state_after_alloc)
calls = Mock()
calls.attach_mock(update_state, "allocated")
calls.attach_mock(get_sources, "sources")
monkeypatch.setattr(scheduler.connector, "update_state_after_alloc", update_state)
monkeypatch.setattr(
scheduler.connector, "get_external_cache_hit_sources", get_sources
)
scheduler.add_request(request)
# A failed allocation is not an admission: nothing is attributed.
with patch.object(scheduler.kv_cache_manager, "allocate_slots", return_value=None):
scheduler.schedule()
get_sources.assert_not_called()
update_state.assert_not_called()
scheduler.schedule()
# Attribution runs once, after the connector has built its load plan.
get_sources.assert_called_once_with(request, num_matched_tokens)
assert [c[0] for c in calls.mock_calls] == ["allocated", "sources"]
connector_stats = scheduler.connector_prefix_cache_stats
assert connector_stats is not None
assert connector_stats.hits == num_matched_tokens
assert connector_stats.hits_by_source == {
CacheHitSource.P2P: block_size,
CacheHitSource.HOST: block_size,
}
if is_async:
# Re-admission after the async load completes is not a new hit.
scheduler.make_stats()
_step_until_kv_transfer_finished(scheduler, [request.request_id])
get_sources.assert_called_once()
connector_stats = scheduler.connector_prefix_cache_stats
assert connector_stats is not None
assert connector_stats.hits_by_source == {}
@pytest.mark.parametrize("is_async", [False, True])
def test_kv_connector_basic(is_async: bool):
"""Test whether Scheduler with KVConnector schedules tokens, allocates