[Metrics] Expose cached prompt tokens by cache tier (#56318)
Signed-off-by: Cam Quilici <cjquilici@gmail.com> Signed-off-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Cam Quilici
Nick Hill
Yifan Qiao
parent
ad0f67a7ec
commit
e37e51dd24
@@ -3,7 +3,7 @@
|
||||
import dataclasses
|
||||
from concurrent.futures import Future
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -60,6 +60,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import (
|
||||
DraftTokenIds,
|
||||
ECConnectorOutput,
|
||||
@@ -2613,6 +2614,65 @@ def test_kv_connector_honors_skip_reading_prefix_cache():
|
||||
assert output.num_scheduled_tokens[scoring.request_id] == BLOCK_SIZE * 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
def test_kv_connector_records_external_cache_hit_sources(monkeypatch, is_async):
|
||||
block_size = 16
|
||||
num_matched_tokens = 2 * block_size
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
use_kv_connector=mock_kv(
|
||||
matched_tokens=num_matched_tokens,
|
||||
is_async=is_async,
|
||||
),
|
||||
block_size=block_size,
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=2 * num_matched_tokens,
|
||||
block_size=block_size,
|
||||
)[0]
|
||||
assert scheduler.connector is not None
|
||||
get_sources = Mock(
|
||||
return_value={CacheHitSource.P2P: block_size, CacheHitSource.HOST: block_size}
|
||||
)
|
||||
update_state = Mock(wraps=scheduler.connector.update_state_after_alloc)
|
||||
calls = Mock()
|
||||
calls.attach_mock(update_state, "allocated")
|
||||
calls.attach_mock(get_sources, "sources")
|
||||
monkeypatch.setattr(scheduler.connector, "update_state_after_alloc", update_state)
|
||||
monkeypatch.setattr(
|
||||
scheduler.connector, "get_external_cache_hit_sources", get_sources
|
||||
)
|
||||
|
||||
scheduler.add_request(request)
|
||||
# A failed allocation is not an admission: nothing is attributed.
|
||||
with patch.object(scheduler.kv_cache_manager, "allocate_slots", return_value=None):
|
||||
scheduler.schedule()
|
||||
get_sources.assert_not_called()
|
||||
update_state.assert_not_called()
|
||||
|
||||
scheduler.schedule()
|
||||
# Attribution runs once, after the connector has built its load plan.
|
||||
get_sources.assert_called_once_with(request, num_matched_tokens)
|
||||
assert [c[0] for c in calls.mock_calls] == ["allocated", "sources"]
|
||||
connector_stats = scheduler.connector_prefix_cache_stats
|
||||
assert connector_stats is not None
|
||||
assert connector_stats.hits == num_matched_tokens
|
||||
assert connector_stats.hits_by_source == {
|
||||
CacheHitSource.P2P: block_size,
|
||||
CacheHitSource.HOST: block_size,
|
||||
}
|
||||
|
||||
if is_async:
|
||||
# Re-admission after the async load completes is not a new hit.
|
||||
scheduler.make_stats()
|
||||
_step_until_kv_transfer_finished(scheduler, [request.request_id])
|
||||
get_sources.assert_called_once()
|
||||
connector_stats = scheduler.connector_prefix_cache_stats
|
||||
assert connector_stats is not None
|
||||
assert connector_stats.hits_by_source == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
def test_kv_connector_basic(is_async: bool):
|
||||
"""Test whether Scheduler with KVConnector schedules tokens, allocates
|
||||
|
||||
Reference in New Issue
Block a user