[Metrics] Expose cached prompt tokens by cache tier (#56318)

Signed-off-by: Cam Quilici <cjquilici@gmail.com>
Signed-off-by: Cam Quilici <cameron@semianalysis.com>
Co-authored-by: Cam Quilici <cameron@semianalysis.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Cameron Quilici
2026-10-06 14:52:05 -04:00
committed by GitHub
co-authored by Cam Quilici Nick Hill Yifan Qiao
parent ad0f67a7ec
commit e37e51dd24
36 changed files with 1005 additions and 44 deletions
+23 -5
View File
@@ -6,14 +6,15 @@ use std::collections::BTreeSet;
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
use vllm_metrics::{ use vllm_metrics::{
EngineLabels, EnginePositionLabels, F64Gauge, Family, HistogramMetric, LoraAdapterNames, CacheHitSourceLabels, EngineLabels, EnginePositionLabels, F64Gauge, Family, HistogramMetric,
LoraInfoLabels, MooncakeOperationCounterFamily, MooncakeOperationHistogramFamily, LoraAdapterNames, LoraInfoLabels, MooncakeOperationCounterFamily,
MooncakeOperationLabels, RequestMetrics, SchedulerLogStatsAccumulator, SchedulerMetrics, MooncakeOperationHistogramFamily, MooncakeOperationLabels, RequestMetrics,
U64Counter, U64Gauge, WaitingReasonLabels, SchedulerLogStatsAccumulator, SchedulerMetrics, U64Counter, U64Gauge, WaitingReasonLabels,
}; };
use crate::protocol::stats::{ use crate::protocol::stats::{
KvConnectorStats, MooncakeStats, MultiConnectorStats, NixlStats, SchedulerStats, CachedTokensBySource, KvConnectorStats, MooncakeStats, MultiConnectorStats, NixlStats,
SchedulerStats,
}; };
use crate::transport::ConnectedEngine; use crate::transport::ConnectedEngine;
@@ -60,6 +61,8 @@ struct SchedulerStatsHandles {
prefix_cache_hits: U64Counter, prefix_cache_hits: U64Counter,
external_prefix_cache_queries: U64Counter, external_prefix_cache_queries: U64Counter,
external_prefix_cache_hits: U64Counter, external_prefix_cache_hits: U64Counter,
/// One handle per `CachedTokensBySource::SOURCES` entry.
prompt_tokens_cached_by_source: [U64Counter; 5],
// Speculative decoding counters. // Speculative decoding counters.
spec_decode_num_drafts: U64Counter, spec_decode_num_drafts: U64Counter,
@@ -165,6 +168,15 @@ fn resolve_scheduler_stats_handles(
.external_prefix_cache_queries .external_prefix_cache_queries
.get_or_create_owned(&labels), .get_or_create_owned(&labels),
external_prefix_cache_hits: metrics.external_prefix_cache_hits.get_or_create_owned(&labels), external_prefix_cache_hits: metrics.external_prefix_cache_hits.get_or_create_owned(&labels),
prompt_tokens_cached_by_source: CachedTokensBySource::SOURCES.map(|source| {
metrics
.prompt_tokens_cached_by_source
.get_or_create_owned(&CacheHitSourceLabels {
model_name: model_name.to_string(),
engine,
source,
})
}),
spec_decode_num_drafts: metrics.spec_decode_num_drafts.get_or_create_owned(&labels), spec_decode_num_drafts: metrics.spec_decode_num_drafts.get_or_create_owned(&labels),
spec_decode_num_draft_tokens: metrics spec_decode_num_draft_tokens: metrics
.spec_decode_num_draft_tokens .spec_decode_num_draft_tokens
@@ -220,6 +232,8 @@ fn record_scheduler_stats_with_handles(handles: &SchedulerStatsHandles, stats: &
// Prefix-cache counters, including the connector-backed external cache path. // Prefix-cache counters, including the connector-backed external cache path.
handles.prefix_cache_queries.inc_by(stats.prefix_cache_stats.base.queries); handles.prefix_cache_queries.inc_by(stats.prefix_cache_stats.base.queries);
handles.prefix_cache_hits.inc_by(stats.prefix_cache_stats.base.hits); handles.prefix_cache_hits.inc_by(stats.prefix_cache_stats.base.hits);
// Local prefix-cache hits are the `device` tier.
handles.prompt_tokens_cached_by_source[0].inc_by(stats.prefix_cache_stats.base.hits);
if let Some(connector_prefix_cache_stats) = &stats.connector_prefix_cache_stats { if let Some(connector_prefix_cache_stats) = &stats.connector_prefix_cache_stats {
handles handles
@@ -228,6 +242,10 @@ fn record_scheduler_stats_with_handles(handles: &SchedulerStatsHandles, stats: &
handles handles
.external_prefix_cache_hits .external_prefix_cache_hits
.inc_by(connector_prefix_cache_stats.base.hits); .inc_by(connector_prefix_cache_stats.base.hits);
let counts = connector_prefix_cache_stats.hits_by_source.counts();
for (handle, count) in handles.prompt_tokens_cached_by_source.iter().zip(counts) {
handle.inc_by(count);
}
} }
// Speculative decoding counters. // Speculative decoding counters.
@@ -21,6 +21,34 @@ pub struct BaseCacheStats {
pub hits: u64, pub hits: u64,
} }
/// Cache-hit tokens per source tier. Python sends a sparse
/// `dict[CacheHitSource, int]`, so absent tiers default to zero.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct CachedTokensBySource {
pub device: u64,
pub host: u64,
pub p2p: u64,
pub disk: u64,
pub external_unspecified: u64,
}
impl CachedTokensBySource {
/// Label values, in the same order as [`Self::counts`].
pub const SOURCES: [&'static str; 5] =
["device", "host", "p2p", "disk", "external_unspecified"];
pub fn counts(&self) -> [u64; 5] {
[
self.device,
self.host,
self.p2p,
self.disk,
self.external_unspecified,
]
}
}
/// Stores prefix cache hit statistics. /// Stores prefix cache hit statistics.
/// - `reset`: Whether `reset_prefix_cache` was invoked. /// - `reset`: Whether `reset_prefix_cache` was invoked.
/// - `queries`: Refers to the number of tokens that were queried. /// - `queries`: Refers to the number of tokens that were queried.
@@ -38,6 +66,9 @@ pub struct PrefixCacheStats {
pub preempted_queries: u64, pub preempted_queries: u64,
/// The `hits` number for preempted requests. /// The `hits` number for preempted requests.
pub preempted_hits: u64, pub preempted_hits: u64,
/// `hits` split by the cache tier that supplied them (connector stats only).
#[serde(default)]
pub hits_by_source: CachedTokensBySource,
} }
/// Single KV cache block eviction sample. /// Single KV cache block eviction sample.
@@ -304,3 +335,31 @@ pub struct SchedulerStats {
/// Estimated MFU/performance stats, when enabled. /// Estimated MFU/performance stats, when enabled.
pub perf_stats: Option<PerfStats>, pub perf_stats: Option<PerfStats>,
} }
#[cfg(test)]
mod tests {
use super::{CachedTokensBySource, PrefixCacheStats};
#[test]
fn prefix_cache_stats_hits_by_source_defaults_and_decodes() {
// Engines without per-source attribution omit the field entirely.
let legacy = serde_json::json!({
"reset": false, "requests": 1, "queries": 8, "hits": 4,
"preempted_requests": 0, "preempted_queries": 0, "preempted_hits": 0
});
let wire = rmp_serde::to_vec_named(&legacy).unwrap();
let stats: PrefixCacheStats = rmp_serde::from_slice(&wire).unwrap();
assert_eq!(stats.base.hits, 4);
assert_eq!(stats.hits_by_source, CachedTokensBySource::default());
let payload = serde_json::json!({
"reset": false, "requests": 1, "queries": 16, "hits": 12,
"preempted_requests": 0, "preempted_queries": 0, "preempted_hits": 0,
"hits_by_source": {"host": 8, "p2p": 4}
});
let wire = rmp_serde::to_vec_named(&payload).unwrap();
let stats: PrefixCacheStats = rmp_serde::from_slice(&wire).unwrap();
assert_eq!(stats.hits_by_source.counts(), [0, 8, 4, 0, 0]);
assert_eq!(CachedTokensBySource::SOURCES[1], "host");
}
}
+16
View File
@@ -86,6 +86,13 @@ pub struct EngineLabels {
pub engine: u32, pub engine: u32,
} }
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct CacheHitSourceLabels {
pub model_name: String,
pub engine: u32,
pub source: &'static str,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] #[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct EnginePositionLabels { pub struct EnginePositionLabels {
pub model_name: String, pub model_name: String,
@@ -236,6 +243,7 @@ pub struct SchedulerMetrics {
pub prefix_cache_hits: Family<EngineLabels, U64Counter>, pub prefix_cache_hits: Family<EngineLabels, U64Counter>,
pub external_prefix_cache_queries: Family<EngineLabels, U64Counter>, pub external_prefix_cache_queries: Family<EngineLabels, U64Counter>,
pub external_prefix_cache_hits: Family<EngineLabels, U64Counter>, pub external_prefix_cache_hits: Family<EngineLabels, U64Counter>,
pub prompt_tokens_cached_by_source: Family<CacheHitSourceLabels, U64Counter>,
// Speculative decoding counters. // Speculative decoding counters.
pub spec_decode_num_drafts: Family<EngineLabels, U64Counter>, pub spec_decode_num_drafts: Family<EngineLabels, U64Counter>,
@@ -339,6 +347,13 @@ impl SchedulerMetrics {
); );
let external_prefix_cache_hits = Family::default(); let external_prefix_cache_hits = Family::default();
let prompt_tokens_cached_by_source = Family::default();
registry.register(
"vllm:prompt_tokens_cached_by_source",
"Prefix-cache hit tokens by the cache tier that supplied them: device (local HBM), host (offloaded to DRAM), disk, p2p (transferred from another vLLM instance) or external_unspecified. Counted at admission; sums to vllm:prefix_cache_hits + vllm:external_prefix_cache_hits.",
prompt_tokens_cached_by_source.clone(),
);
registry.register( registry.register(
"vllm:external_prefix_cache_hits", "vllm:external_prefix_cache_hits",
"External prefix cache hits from KV connector cross-instance cache sharing, in terms of number of cached tokens.", "External prefix cache hits from KV connector cross-instance cache sharing, in terms of number of cached tokens.",
@@ -528,6 +543,7 @@ impl SchedulerMetrics {
prefix_cache_hits, prefix_cache_hits,
external_prefix_cache_queries, external_prefix_cache_queries,
external_prefix_cache_hits, external_prefix_cache_hits,
prompt_tokens_cached_by_source,
spec_decode_num_drafts, spec_decode_num_drafts,
spec_decode_num_draft_tokens, spec_decode_num_draft_tokens,
spec_decode_num_accepted_tokens, spec_decode_num_accepted_tokens,
+61 -1
View File
@@ -3,7 +3,7 @@
import dataclasses import dataclasses
from concurrent.futures import Future from concurrent.futures import Future
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import Mock from unittest.mock import Mock, patch
import numpy as np import numpy as np
import pytest import pytest
@@ -60,6 +60,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheGroupSpec, KVCacheGroupSpec,
MambaSpec, MambaSpec,
) )
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import ( from vllm.v1.outputs import (
DraftTokenIds, DraftTokenIds,
ECConnectorOutput, ECConnectorOutput,
@@ -2613,6 +2614,65 @@ def test_kv_connector_honors_skip_reading_prefix_cache():
assert output.num_scheduled_tokens[scoring.request_id] == BLOCK_SIZE * 4 assert output.num_scheduled_tokens[scoring.request_id] == BLOCK_SIZE * 4
@pytest.mark.parametrize("is_async", [False, True])
def test_kv_connector_records_external_cache_hit_sources(monkeypatch, is_async):
block_size = 16
num_matched_tokens = 2 * block_size
scheduler = create_scheduler(
enable_prefix_caching=True,
use_kv_connector=mock_kv(
matched_tokens=num_matched_tokens,
is_async=is_async,
),
block_size=block_size,
)
request = create_requests(
num_requests=1,
num_tokens=2 * num_matched_tokens,
block_size=block_size,
)[0]
assert scheduler.connector is not None
get_sources = Mock(
return_value={CacheHitSource.P2P: block_size, CacheHitSource.HOST: block_size}
)
update_state = Mock(wraps=scheduler.connector.update_state_after_alloc)
calls = Mock()
calls.attach_mock(update_state, "allocated")
calls.attach_mock(get_sources, "sources")
monkeypatch.setattr(scheduler.connector, "update_state_after_alloc", update_state)
monkeypatch.setattr(
scheduler.connector, "get_external_cache_hit_sources", get_sources
)
scheduler.add_request(request)
# A failed allocation is not an admission: nothing is attributed.
with patch.object(scheduler.kv_cache_manager, "allocate_slots", return_value=None):
scheduler.schedule()
get_sources.assert_not_called()
update_state.assert_not_called()
scheduler.schedule()
# Attribution runs once, after the connector has built its load plan.
get_sources.assert_called_once_with(request, num_matched_tokens)
assert [c[0] for c in calls.mock_calls] == ["allocated", "sources"]
connector_stats = scheduler.connector_prefix_cache_stats
assert connector_stats is not None
assert connector_stats.hits == num_matched_tokens
assert connector_stats.hits_by_source == {
CacheHitSource.P2P: block_size,
CacheHitSource.HOST: block_size,
}
if is_async:
# Re-admission after the async load completes is not a new hit.
scheduler.make_stats()
_step_until_kv_transfer_finished(scheduler, [request.request_id])
get_sources.assert_called_once()
connector_stats = scheduler.connector_prefix_cache_stats
assert connector_stats is not None
assert connector_stats.hits_by_source == {}
@pytest.mark.parametrize("is_async", [False, True]) @pytest.mark.parametrize("is_async", [False, True])
def test_kv_connector_basic(is_async: bool): def test_kv_connector_basic(is_async: bool):
"""Test whether Scheduler with KVConnector schedules tokens, allocates """Test whether Scheduler with KVConnector schedules tokens, allocates
@@ -77,6 +77,7 @@ from vllm.v1.kv_offload.base import (
make_offload_key, make_offload_key,
) )
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
from vllm.v1.request import RequestStatus from vllm.v1.request import RequestStatus
@@ -197,14 +198,18 @@ def test_swa_offload_window_covers_unaligned_hit(
manager.complete_load(job.keys, state.req_context) manager.complete_load(job.keys, state.req_context)
def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler: def _make_partial_tail_scheduler(
recurrent_only: bool = False,
) -> OffloadingConnectorScheduler:
vllm_config = _make_vllm_config(extra_config={"self_describing_kv_events": True}) vllm_config = _make_vllm_config(extra_config={"self_describing_kv_events": True})
vllm_config.cache_config.prefix_match_unit = 4 vllm_config.cache_config.prefix_match_unit = None if recurrent_only else 4
vllm_config.speculative_config = None vllm_config.speculative_config = None
vllm_config.kv_events_config = KVEventsConfig( vllm_config.kv_events_config = KVEventsConfig(
enable_kv_cache_events=True, publisher="null" enable_kv_cache_events=True, publisher="null"
) )
kv_cache_config = _make_mamba_hybrid_kv_cache_config() kv_cache_config = _make_mamba_hybrid_kv_cache_config()
if recurrent_only:
kv_cache_config.kv_cache_groups = kv_cache_config.kv_cache_groups[1:]
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config)) spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
return OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config) return OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
@@ -576,6 +581,284 @@ def test_recurrent_group_unhashed_block_does_not_truncate_load_boundary():
assert 42 in loaded assert 42 in loaded
@pytest.mark.parametrize(
("recurrent_source", "expected"),
[
("host", {CacheHitSource.P2P: 16, CacheHitSource.HOST: 12}),
("p2p", {CacheHitSource.P2P: 28}),
(None, {CacheHitSource.P2P: 16, CacheHitSource.HOST: 12}),
],
)
def test_external_cache_hit_sources_preserve_partial_tail_origin(
recurrent_source, expected
):
"""Recurrent state covers the whole prefix; the outermost tier wins."""
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(scheduler)
req_status = scheduler._req_status["req"]
req_status.update_offload_keys()
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler._lookup(req_status) == 28
def get_load_source(key, req_context):
if get_offload_group_idx(key) == 1:
assert recurrent_source is not None
return recurrent_source
block_hash = get_offload_block_hash(key)
return "p2p" if block_hash == b"h3" else "host"
scheduler.manager.get_load_source.side_effect = get_load_source
recurrent_block = KVCacheBlock(41)
if recurrent_source is None:
recurrent_block.set_block_hash(
make_block_hash_with_group_id(BlockHash(b"cached"), 1), 12
)
scheduler.update_state_after_alloc(
request,
KVCacheBlocks(
(
[KVCacheBlock(31), KVCacheBlock(32)],
[KVCacheBlock(0, is_null=True), recurrent_block],
)
),
num_external_tokens=28,
)
assert req_status.partial_tail_boundary is None
assert scheduler.get_external_cache_hit_sources(request, 28) == expected
@pytest.mark.parametrize(
("second_source", "expected"),
[
("host", {CacheHitSource.HOST: 4, CacheHitSource.DISK: 4}),
("p2p", {CacheHitSource.P2P: 4, CacheHitSource.DISK: 4}),
],
)
def test_external_cache_hit_sources_reconcile_different_group_chunk_sizes(
request_runner,
second_source,
expected,
):
"""Groups with different chunk sizes overlap; the outermost tier wins."""
block_size = 4
groups = [
KVCacheGroupSpec(
[f"layer{group_idx}"],
FullAttentionSpec(
block_size=block_size * (group_idx + 1),
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
)
for group_idx in range(2)
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=False,
kv_cache_groups=groups,
)
scheduler = runner.connector_scheduler
request = MagicMock(request_id="mixed")
req_status = RequestOffloadState(
config=scheduler.config,
req=request,
req_context=ReqContext(req_id=request.request_id),
offloading_context=RequestOffloadingContext(policy=OffloadPolicy.CHUNK_LEVEL),
)
for group_idx, group_state in enumerate(req_status.group_states):
group_state.offload_keys = [
make_offload_key(f"h{chunk_idx}".encode(), group_idx)
for chunk_idx in range(2 // (group_idx + 1))
]
scheduler._req_status[request.request_id] = req_status
runner.manager.get_load_source.side_effect = lambda key, req_context: (
("host" if get_offload_block_hash(key) == b"h0" else "disk")
if get_offload_group_idx(key) == 0
else second_source
)
scheduler.update_state_after_alloc(
request,
KVCacheBlocks(([KVCacheBlock(1), KVCacheBlock(2)], [KVCacheBlock(3)])),
2 * block_size,
)
assert scheduler.get_external_cache_hit_sources(request, 2 * block_size) == expected
@pytest.mark.parametrize(
("ranges", "expected"),
[
pytest.param(
[(8, 12, "host"), (0, 8, "host"), (4, 10, "host")],
{CacheHitSource.HOST: 8, CacheHitSource.EXTERNAL_UNSPECIFIED: 4},
id="overlapping-same-source-and-adjacent-endpoints",
),
pytest.param(
[(10, 20, "p2p"), (6, 10, "disk"), (0, 8, "host"), (4, 12, "host")],
{CacheHitSource.HOST: 2, CacheHitSource.DISK: 4, CacheHitSource.P2P: 6},
id="nested-tiers-and-clipped-ranges",
),
pytest.param(
[(0, 4, "disk"), (16, 20, "disk"), (8, 8, "disk")],
{CacheHitSource.EXTERNAL_UNSPECIFIED: 12},
id="no-covering-ranges",
),
],
)
def test_external_cache_hit_sources_range_boundaries(ranges, expected):
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(scheduler)
state = scheduler._req_status[request.request_id]
state.num_locally_computed_tokens = 4
for idx, (lo, hi, source) in enumerate(ranges):
key = make_offload_key(str(idx).encode(), 0)
state.load_key_ranges.append((lo, hi, key))
scheduler.manager.get_load_source.side_effect = lambda key, ctx: ranges[
int(get_offload_block_hash(key))
][2]
result = scheduler.get_external_cache_hit_sources(request, 12)
assert result == expected
assert sum(result.values()) == 12
@pytest.mark.parametrize("source", ["host", "disk", "p2p", "external_unspecified"])
def test_external_cache_hit_sources_recurrent_only_state(source):
scheduler = _make_partial_tail_scheduler(recurrent_only=True)
request = _make_partial_tail_request(scheduler)
request.num_prompt_tokens = request.num_tokens = 64
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(4)]
request.all_token_ids = list(range(64))
state = scheduler._req_status[request.request_id]
state.update_offload_keys()
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler._lookup(state) == 48
scheduler.manager.get_load_source.return_value = source
scheduler.update_state_after_alloc(
request,
KVCacheBlocks(
(
[
KVCacheBlock(0, is_null=True),
KVCacheBlock(0, is_null=True),
KVCacheBlock(41),
],
)
),
48,
)
assert scheduler.get_external_cache_hit_sources(request, 48) == {
CacheHitSource(source): 48
}
scheduler.manager.get_load_source.assert_called_once()
@pytest.mark.parametrize("sparse_kind", ["sliding", "chunked_local"])
@pytest.mark.parametrize("blocks_per_chunk", [1, 2])
@pytest.mark.parametrize("local_tokens", [0, 4])
@pytest.mark.parametrize("full_attention", [False, True])
@pytest.mark.parametrize(
"sparse_source", ["host", "disk", "p2p", "external_unspecified", "mixed"]
)
def test_external_cache_hit_sources_use_loaded_attention_ranges(
sparse_kind,
blocks_per_chunk,
local_tokens,
full_attention,
sparse_source,
):
"""SWA/local attention affects only loaded positions, not omitted history."""
block_size = 4
chunk_size = block_size * blocks_per_chunk
spec_kwargs = dict(
block_size=block_size, num_kv_heads=1, head_size=1, dtype=torch.float32
)
sparse_spec = (
SlidingWindowSpec(**spec_kwargs, sliding_window=2 * chunk_size)
if sparse_kind == "sliding"
else ChunkedLocalAttentionSpec(
**spec_kwargs, attention_chunk_size=2 * chunk_size
)
)
groups = [KVCacheGroupSpec(["sparse"], sparse_spec)]
if full_attention:
groups.insert(0, KVCacheGroupSpec(["full"], FullAttentionSpec(**spec_kwargs)))
vllm_config = _make_vllm_config(extra_config={"block_size": chunk_size})
vllm_config.cache_config.block_size = block_size
vllm_config.speculative_config = None
kv_cache_config = KVCacheConfig(
num_blocks=100,
kv_cache_tensors=[],
kv_cache_groups=groups,
)
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
scheduler = OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
request = MagicMock(request_id="sparse")
req_status = RequestOffloadState(
config=scheduler.config,
req=request,
req_context=ReqContext(req_id=request.request_id),
offloading_context=RequestOffloadingContext(policy=OffloadPolicy.CHUNK_LEVEL),
num_locally_computed_tokens=local_tokens,
)
scheduler._req_status[request.request_id] = req_status
block_groups = []
for group_idx, state in enumerate(req_status.group_states):
state.offload_keys = [
make_offload_key(str(i).encode(), group_idx) for i in range(4)
]
block_groups.append(
[
KVCacheBlock(
0
if group_idx == len(groups) - 1 and i < 2 * blocks_per_chunk
else 1 + group_idx * 100 + i,
is_null=group_idx == len(groups) - 1 and i < 2 * blocks_per_chunk,
)
for i in range(4 * blocks_per_chunk)
]
)
def get_load_source(key, req_context):
chunk_idx = int(get_offload_block_hash(key))
if get_offload_group_idx(key) == len(groups) - 1:
assert chunk_idx >= 2 # Earlier sparse keys were not loaded.
if sparse_source == "mixed":
return "host" if chunk_idx == 2 else "disk"
return sparse_source
return "host"
scheduler.manager.get_load_source.side_effect = get_load_source
count = 4 * chunk_size - local_tokens
scheduler.update_state_after_alloc(
request, KVCacheBlocks(tuple(block_groups)), count
)
expected: dict[CacheHitSource, int] = {}
def expect(source: CacheHitSource, num_tokens: int) -> None:
expected[source] = expected.get(source, 0) + num_tokens
expect(
CacheHitSource.HOST if full_attention else CacheHitSource.EXTERNAL_UNSPECIFIED,
2 * chunk_size - local_tokens,
)
for chunk_idx in range(2, 4):
source = (
("host" if chunk_idx == 2 else "disk")
if sparse_source == "mixed"
else sparse_source
)
expect(CacheHitSource(source), chunk_size)
result = scheduler.get_external_cache_hit_sources(request, count)
assert result == expected
assert sum(result.values()) == count
assert scheduler.get_external_cache_hit_sources(request, 0) == {}
def test_partial_lookup_requires_every_cache_group(): def test_partial_lookup_requires_every_cache_group():
scheduler = _make_partial_tail_scheduler() scheduler = _make_partial_tail_scheduler()
_make_partial_tail_request(scheduler) _make_partial_tail_request(scheduler)
@@ -1400,7 +1683,6 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
# are within the window → loads blocks 1,2 # are within the window → loads blocks 1,2
expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)), expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)),
) )
runner.manager.touch.assert_not_called() runner.manager.touch.assert_not_called()
# 3 blocks are hit on GPU [0, 1, 2] # 3 blocks are hit on GPU [0, 1, 2]
@@ -57,6 +57,7 @@ from vllm.v1.kv_offload.base import (
make_offload_key, make_offload_key,
) )
from vllm.v1.kv_offload.config import OffloadingConfig from vllm.v1.kv_offload.config import OffloadingConfig
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.request import Request from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager from vllm.v1.structured_output import StructuredOutputManager
@@ -129,6 +130,7 @@ class MockOffloadingSpec(OffloadingSpec):
self.manager = MagicMock(spec=OffloadingManager) self.manager = MagicMock(spec=OffloadingManager)
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys) self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
self.manager.lookup.return_value = LookupResult.MISS self.manager.lookup.return_value = LookupResult.MISS
self.manager.get_load_source.return_value = CacheHitSource.EXTERNAL_UNSPECIFIED
self.manager.get_stats.return_value = None self.manager.get_stats.return_value = None
self.manager.on_new_request.return_value = RequestOffloadingContext() self.manager.on_new_request.return_value = RequestOffloadingContext()
self.handler = MockOffloadingWorker() self.handler = MockOffloadingWorker()
@@ -8,7 +8,6 @@ prefix cache hash table so future requests cannot match and reuse corrupted data
""" """
from collections.abc import Callable from collections.abc import Callable
from unittest.mock import Mock
import pytest import pytest
@@ -16,6 +15,7 @@ from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.request import Request, RequestStatus from vllm.v1.request import Request, RequestStatus
from .utils import ( from .utils import (
create_mock_connector,
create_model_runner_output, create_model_runner_output,
create_request, create_request,
create_scheduler, create_scheduler,
@@ -73,7 +73,7 @@ def test_invalid_blocks_evicted_prevents_cache_pollution(
} }
# mock connector indicating sync load # mock connector indicating sync load
fail_scheduler.connector = Mock() fail_scheduler.connector = create_mock_connector()
fail_scheduler.connector.get_num_new_matched_tokens.side_effect = ( fail_scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, False) _make_get_num_new_matched_tokens(req_num_new_matched_tokens, False)
) )
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable from collections.abc import Callable
from unittest.mock import Mock
import pytest import pytest
@@ -10,6 +9,7 @@ from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.request import FinishReason, Request, RequestStatus from vllm.v1.request import FinishReason, Request, RequestStatus
from .utils import ( from .utils import (
create_mock_connector,
create_model_runner_output, create_model_runner_output,
create_request, create_request,
create_scheduler, create_scheduler,
@@ -56,7 +56,7 @@ def test_error_propagation_sync_load(fail_scheduler: Scheduler):
request.request_id: num_external_computed_tokens, request.request_id: num_external_computed_tokens,
} }
fail_scheduler.connector = Mock() fail_scheduler.connector = create_mock_connector()
fail_scheduler.connector.get_num_new_matched_tokens.side_effect = ( fail_scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, False) _make_get_num_new_matched_tokens(req_num_new_matched_tokens, False)
) )
@@ -110,7 +110,7 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler):
request.request_id: num_external_computed_tokens, request.request_id: num_external_computed_tokens,
} }
fail_scheduler.connector = Mock() fail_scheduler.connector = create_mock_connector()
fail_scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,) fail_scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
fail_scheduler.connector.get_num_new_matched_tokens.side_effect = ( fail_scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, True) _make_get_num_new_matched_tokens(req_num_new_matched_tokens, True)
@@ -11,7 +11,6 @@ These tests verify correct behavior in three scenarios:
""" """
from collections.abc import Callable from collections.abc import Callable
from unittest.mock import Mock
import pytest import pytest
@@ -19,6 +18,7 @@ from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.request import FinishReason, Request, RequestStatus from vllm.v1.request import FinishReason, Request, RequestStatus
from .utils import ( from .utils import (
create_mock_connector,
create_model_runner_output, create_model_runner_output,
create_request, create_request,
create_scheduler, create_scheduler,
@@ -83,7 +83,7 @@ def test_sync_recompute_blocks_not_freed_for_running_requests(
} }
# mock connector indicating sync load # mock connector indicating sync load
recompute_scheduler.connector = Mock() recompute_scheduler.connector = create_mock_connector()
recompute_scheduler.connector.get_num_new_matched_tokens.side_effect = ( recompute_scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, False) _make_get_num_new_matched_tokens(req_num_new_matched_tokens, False)
) )
@@ -206,7 +206,7 @@ def test_sync_fail_invalid_blocks_evicted(fail_scheduler: Scheduler):
} }
# mock connector indicating sync load # mock connector indicating sync load
fail_scheduler.connector = Mock() fail_scheduler.connector = create_mock_connector()
fail_scheduler.connector.get_num_new_matched_tokens.side_effect = ( fail_scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, False) _make_get_num_new_matched_tokens(req_num_new_matched_tokens, False)
) )
@@ -323,7 +323,7 @@ def test_async_recompute_blocks_not_cached_when_invalid(
} }
# mock connector indicating async load # mock connector indicating async load
recompute_scheduler.connector = Mock() recompute_scheduler.connector = create_mock_connector()
recompute_scheduler.connector.get_num_new_matched_tokens.side_effect = ( recompute_scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, True) _make_get_num_new_matched_tokens(req_num_new_matched_tokens, True)
) )
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable from collections.abc import Callable
from unittest.mock import Mock
import pytest import pytest
@@ -10,6 +9,7 @@ from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.request import Request, RequestStatus from vllm.v1.request import Request, RequestStatus
from .utils import ( from .utils import (
create_mock_connector,
create_model_runner_output, create_model_runner_output,
create_request, create_request,
create_scheduler, create_scheduler,
@@ -41,7 +41,7 @@ def test_failed_receive_completion_honors_failure_policy(policy):
scheduler = create_scheduler(create_vllm_config(kv_load_failure_policy=policy)) scheduler = create_scheduler(create_vllm_config(kv_load_failure_policy=policy))
request = create_request(num_tokens=3 * scheduler.block_size) request = create_request(num_tokens=3 * scheduler.block_size)
scheduler.add_request(request) scheduler.add_request(request)
scheduler.connector = Mock() scheduler.connector = create_mock_connector()
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,) scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
scheduler.connector.get_num_new_matched_tokens.side_effect = [ scheduler.connector.get_num_new_matched_tokens.side_effect = [
(2 * scheduler.block_size, True), (2 * scheduler.block_size, True),
@@ -103,7 +103,7 @@ def test_async_load_failure(
request3.request_id: num_external_computed_tokens, request3.request_id: num_external_computed_tokens,
} }
scheduler.connector = Mock() scheduler.connector = create_mock_connector()
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,) scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
scheduler.connector.get_num_new_matched_tokens.side_effect = ( scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True) _make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True)
@@ -181,7 +181,7 @@ def test_sync_load_failure(
request3.request_id: num_external_computed_tokens, request3.request_id: num_external_computed_tokens,
} }
scheduler.connector = Mock() scheduler.connector = create_mock_connector()
scheduler.connector.get_num_new_matched_tokens.side_effect = ( scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=False) _make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=False)
) )
@@ -262,7 +262,7 @@ def test_sync_load_failure_with_shared_blocks(
request1.request_id: num_external_computed_tokens, request1.request_id: num_external_computed_tokens,
} }
scheduler.connector = Mock() scheduler.connector = create_mock_connector()
scheduler.connector.get_num_new_matched_tokens.side_effect = ( scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=False) _make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=False)
) )
@@ -333,7 +333,7 @@ def test_async_progressive_load_failure(
request.request_id: num_external_computed_tokens, request.request_id: num_external_computed_tokens,
} }
scheduler.connector = Mock() scheduler.connector = create_mock_connector()
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,) scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
scheduler.connector.get_num_new_matched_tokens.side_effect = ( scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True) _make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True)
@@ -38,6 +38,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheTensor, KVCacheTensor,
MambaSpec, MambaSpec,
) )
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput from vllm.v1.outputs import KVConnectorOutput
from .utils import create_vllm_config from .utils import create_vllm_config
@@ -208,6 +209,29 @@ def test_scheduler_role_initializes_store_scheduler_only():
mock_scheduler.return_value.bind_gpu_block_pool.assert_called_once_with(block_pool) mock_scheduler.return_value.bind_gpu_block_pool.assert_called_once_with(block_pool)
def test_scheduler_reports_mooncake_cache_source():
vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config()
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"connector.MooncakeStoreScheduler"
),
):
connector = mooncake_store_connector.MooncakeStoreConnector(
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
)
# Mooncake Store is an external cache service. Its internal memory and
# local-storage policy is intentionally hidden behind that stable label.
assert connector.get_external_cache_hit_sources(
None, # type: ignore[arg-type]
32,
) == {CacheHitSource.EXTERNAL_UNSPECIFIED: 32}
def test_worker_methods_delegate_to_store_worker(): def test_worker_methods_delegate_to_store_worker():
vllm_config = _make_vllm_config() vllm_config = _make_vllm_config()
kv_cache_config = _make_kv_cache_config() kv_cache_config = _make_kv_cache_config()
@@ -34,6 +34,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
NixlKVConnectorStats, NixlKVConnectorStats,
) )
from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput, KVConnectorWorkerMetadata from vllm.v1.outputs import KVConnectorOutput, KVConnectorWorkerMetadata
MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct" MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
@@ -246,6 +247,18 @@ def test_multi_connector_rejects_multiple_mem_pool_contexts(mc: MultiConnector):
mc.get_mem_pool_context() mc.get_mem_pool_context()
def test_cache_hit_sources_delegate_to_selected_connector(mc: MultiConnector):
request = MagicMock(request_id="request")
mc._requests_to_connector[request.request_id] = 1
sources = {CacheHitSource.DISK: 32}
mc._connectors[1].get_external_cache_hit_sources.return_value = sources
assert mc.get_external_cache_hit_sources(request, 32) is sources
mc._connectors[1].get_external_cache_hit_sources.assert_called_once_with(
request, 32
)
# Helper function to compare directories recursively # Helper function to compare directories recursively
def _compare_directories(dir1: Path, dir2: Path) -> bool: def _compare_directories(dir1: Path, dir2: Path) -> bool:
"""Compares two directories recursively for identical content.""" """Compares two directories recursively for identical content."""
@@ -36,6 +36,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheTensor, KVCacheTensor,
SlidingWindowSpec, SlidingWindowSpec,
) )
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request from vllm.v1.request import Request
from vllm.v1.simple_kv_offload.metadata import ( from vllm.v1.simple_kv_offload.metadata import (
@@ -232,6 +233,9 @@ def test_nixl_wins_load_over_cpu_offload():
assert not sched_out.has_sync_kv_loads assert not sched_out.has_sync_kv_loads
assert mc._requests_to_connector[request.request_id] == 0 assert mc._requests_to_connector[request.request_id] == 0
assert mc.get_external_cache_hit_sources(request, 2 * BLOCK_SIZE) == {
CacheHitSource.P2P: 2 * BLOCK_SIZE
}
meta = sched_out.kv_connector_metadata meta = sched_out.kv_connector_metadata
assert isinstance(meta, MultiKVConnectorMetadata) assert isinstance(meta, MultiKVConnectorMetadata)
@@ -302,6 +306,9 @@ def test_cpu_offload_wins_when_nixl_has_no_match():
assert hit_tokens is not None and hit_tokens > 0 assert hit_tokens is not None and hit_tokens > 0
assert mc._requests_to_connector[req2.request_id] == 1 assert mc._requests_to_connector[req2.request_id] == 1
assert is_async is True assert is_async is True
assert mc.get_external_cache_hit_sources(req2, hit_tokens) == {
CacheHitSource.HOST: hit_tokens
}
@pytest.mark.parametrize("swa_enabled", [False, True], ids=["fa_only", "fa_sw"]) @pytest.mark.parametrize("swa_enabled", [False, True], ids=["fa_only", "fa_sw"])
+10
View File
@@ -6,6 +6,7 @@ from collections.abc import Callable
from dataclasses import dataclass from dataclasses import dataclass
from itertools import chain, count from itertools import chain, count
from typing import Any, Literal from typing import Any, Literal
from unittest.mock import Mock
import torch import torch
@@ -42,6 +43,7 @@ from vllm.v1.kv_cache_interface import (
MambaSpec, MambaSpec,
SlidingWindowSpec, SlidingWindowSpec,
) )
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
from vllm.v1.request import Request from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager from vllm.v1.structured_output import StructuredOutputManager
@@ -49,6 +51,14 @@ from vllm.v1.structured_output import StructuredOutputManager
EOS_TOKEN_ID = 50256 EOS_TOKEN_ID = 50256
def create_mock_connector() -> Mock:
connector = Mock()
connector.get_external_cache_hit_sources.side_effect = lambda request, num_tokens: (
{CacheHitSource.EXTERNAL_UNSPECIFIED: num_tokens} if num_tokens else {}
)
return connector
def assert_scheduler_empty(scheduler: Scheduler): def assert_scheduler_empty(scheduler: Scheduler):
"""Confirm the scheduler is "empty" - i.e. no leaks.""" """Confirm the scheduler is "empty" - i.e. no leaks."""
# Scheduler Metadata. # Scheduler Metadata.
+18 -4
View File
@@ -17,6 +17,7 @@ import pytest
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
from vllm.v1.metrics.cache_hit_source import CacheHitSource
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Fixtures # Fixtures
@@ -43,12 +44,22 @@ def _make_mock_args():
def test_pre_registered_tiers_can_be_imported(): def test_pre_registered_tiers_can_be_imported():
"""CI sentinel: registered tiers import and yield SecondaryTierManager.""" """CI sentinel: registered tiers import and yield SecondaryTierManager."""
# Derived from each tier's ``medium``; P2P has none and sets its own.
expected_sources = {
"example": CacheHitSource.HOST,
"fs": CacheHitSource.DISK,
"obj": CacheHitSource.DISK,
"p2p": CacheHitSource.P2P,
}
for tier_type in SecondaryTierFactory._registry: for tier_type in SecondaryTierFactory._registry:
# KVCR is an optional external dependency and may not be installed. # KVCR is an optional external dependency and may not be installed.
if tier_type == "kvcr" and find_spec("kvcr") is None: if tier_type == "kvcr" and find_spec("kvcr") is None:
continue continue
cls = SecondaryTierFactory._registry[tier_type]() cls = SecondaryTierFactory._registry[tier_type]()
assert issubclass(cls, SecondaryTierManager) assert issubclass(cls, SecondaryTierManager)
assert cls.cache_hit_source == expected_sources.get(
tier_type, CacheHitSource.EXTERNAL_UNSPECIFIED
)
def test_example_tier_registered(): def test_example_tier_registered():
@@ -95,7 +106,8 @@ def test_create_multiple_tiers():
assert all(isinstance(tier, ExampleSecondaryTierManager) for tier in tiers) assert all(isinstance(tier, ExampleSecondaryTierManager) for tier in tiers)
def test_register_new_tier_type(): @pytest.mark.parametrize("tier_type", ["custom_tier", "cpu", "nvme"])
def test_register_new_tier_type(tier_type):
"""Verify that new tier types can be registered and created. """Verify that new tier types can be registered and created.
This is how external projects add custom secondary tiers This is how external projects add custom secondary tiers
@@ -103,20 +115,22 @@ def test_register_new_tier_type():
""" """
# Register a new tier type (reuse example manager for simplicity) # Register a new tier type (reuse example manager for simplicity)
SecondaryTierFactory.register_tier( SecondaryTierFactory.register_tier(
"custom_tier", tier_type,
"vllm.v1.kv_offload.tiering.example.manager", "vllm.v1.kv_offload.tiering.example.manager",
"ExampleSecondaryTierManager", "ExampleSecondaryTierManager",
) )
primary_kv_view, offloading_spec = _make_mock_args() primary_kv_view, offloading_spec = _make_mock_args()
tier = SecondaryTierFactory.create_secondary_tier( tier = SecondaryTierFactory.create_secondary_tier(
{"type": "custom_tier", "custom_param": 99}, {"type": tier_type, "custom_param": 99},
primary_kv_view, primary_kv_view,
offloading_spec, offloading_spec,
) )
assert tier.tier_type == "custom_tier" assert tier.tier_type == tier_type
assert isinstance(tier, ExampleSecondaryTierManager) assert isinstance(tier, ExampleSecondaryTierManager)
# Attribution follows the tier's medium, never its registration name.
assert tier.cache_hit_source is CacheHitSource.HOST
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -53,6 +53,7 @@ from vllm.v1.kv_offload.tiering.manager import (
TieringOffloadingManager, TieringOffloadingManager,
) )
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
from vllm.v1.metrics.cache_hit_source import CacheHitSource
_CTX = ReqContext(req_id="test") _CTX = ReqContext(req_id="test")
_MOCK_OFFLOADING_SPEC = MagicMock() _MOCK_OFFLOADING_SPEC = MagicMock()
@@ -500,8 +501,13 @@ class TestTieringOffloadingManager:
# Lookup should find all chunks in primary # Lookup should find all chunks in primary
assert count_hits(self.manager, chunks) == 3 assert count_hits(self.manager, chunks) == 3
def test_promotion_from_secondary(self, manager_setup): @pytest.mark.parametrize(
"source",
[CacheHitSource.EXTERNAL_UNSPECIFIED, CacheHitSource.DISK, CacheHitSource.P2P],
)
def test_promotion_from_secondary(self, manager_setup, monkeypatch, source):
"""Test promotion of chunks from secondary to primary tier.""" """Test promotion of chunks from secondary to primary tier."""
monkeypatch.setattr(ExampleSecondaryTierManager, "cache_hit_source", source)
chunks = to_keys(range(3)) chunks = to_keys(range(3))
# Manually add chunks to secondary tier (simulate previous cascade) # Manually add chunks to secondary tier (simulate previous cascade)
@@ -525,6 +531,27 @@ class TestTieringOffloadingManager:
# Next lookup should succeed # Next lookup should succeed
assert count_hits(self.manager, chunks) == 3 assert count_hits(self.manager, chunks) == 3
# The request that caused the promotion retains the secondary origin.
assert all(
self.manager.get_load_source(chunk, _CTX) == source for chunk in chunks
)
# A later request sees the chunks as ordinary host-memory primary hits.
later_context = ReqContext(req_id="later")
self.manager.on_new_request(later_context)
assert all(
self.manager.lookup(chunk, later_context) is LookupResult.HIT
for chunk in chunks
)
assert all(
self.manager.get_load_source(chunk, later_context) == "host"
for chunk in chunks
)
@pytest.mark.parametrize(
"source",
[CacheHitSource.EXTERNAL_UNSPECIFIED, CacheHitSource.DISK, CacheHitSource.P2P],
)
@pytest.mark.parametrize( @pytest.mark.parametrize(
("successful_indices", "expected_results"), ("successful_indices", "expected_results"),
[ [
@@ -540,8 +567,9 @@ class TestTieringOffloadingManager:
ids=["partial", "legacy-full-failure"], ids=["partial", "legacy-full-failure"],
) )
def test_failed_promotion_keeps_only_successful_chunks( def test_failed_promotion_keeps_only_successful_chunks(
self, manager_setup, successful_indices, expected_results self, manager_setup, monkeypatch, source, successful_indices, expected_results
): ):
monkeypatch.setattr(ExampleSecondaryTierManager, "cache_hit_source", source)
chunks = to_keys(range(3)) chunks = to_keys(range(3))
for chunk in chunks: for chunk in chunks:
self.secondary_tier1.chunks[chunk] = True self.secondary_tier1.chunks[chunk] = True
@@ -571,6 +599,14 @@ class TestTieringOffloadingManager:
assert [ assert [
self.primary_tier.lookup(chunk, _CTX) for chunk in chunks self.primary_tier.lookup(chunk, _CTX) for chunk in chunks
] == expected_results ] == expected_results
expected_source_keys = (
set()
if successful_indices is None
else {chunks[i] for i in successful_indices}
)
assert set(self.manager._request_load_sources[_CTX.req_id]) == (
expected_source_keys
)
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup): def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
"""Resolved lookups report one sync delay sample per tier and chunk.""" """Resolved lookups report one sync delay sample per tier and chunk."""
@@ -1315,6 +1351,7 @@ class TestTieringOffloadingManager:
# Orchestrator state cleared. # Orchestrator state cleared.
assert self.manager._jobs == {} assert self.manager._jobs == {}
assert self.manager._pending_load_submissions == {} assert self.manager._pending_load_submissions == {}
assert self.manager._request_load_sources == {}
assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id} assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id}
assert self.manager._processed_jobs_this_step is False assert self.manager._processed_jobs_this_step is False
+93
View File
@@ -1,10 +1,13 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.v1.core.sched.output import ScheduledEncoderInputStats, SchedulerOutput from vllm.v1.core.sched.output import ScheduledEncoderInputStats, SchedulerOutput
from vllm.v1.engine import EngineCoreOutputs, FinishReason from vllm.v1.engine import EngineCoreOutputs, FinishReason
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.metrics.stats import ( from vllm.v1.metrics.stats import (
IterationStats, IterationStats,
PrefillStats, PrefillStats,
PrefixCacheStats,
PromptTokenStats, PromptTokenStats,
RequestStateStats, RequestStateStats,
SchedulerIterationDetails, SchedulerIterationDetails,
@@ -45,6 +48,96 @@ def test_scheduler_iteration_details_serialization():
assert decoded.scheduler_stats.iteration_details == iteration_details assert decoded.scheduler_stats.iteration_details == iteration_details
def test_prefix_cache_stats_hits_by_source_accumulate():
stats = PrefixCacheStats()
stats.record(
num_tokens=8,
num_hits=5,
preempted=False,
hits_by_source={CacheHitSource.HOST: 3, CacheHitSource.DISK: 2},
)
stats.record(
num_tokens=8,
num_hits=6,
preempted=False,
hits_by_source={CacheHitSource.P2P: 5, CacheHitSource.HOST: 1},
)
# Zero counts never create an entry.
stats.record(
num_tokens=8,
num_hits=4,
preempted=False,
hits_by_source={CacheHitSource.DISK: 4, CacheHitSource.P2P: 0},
)
assert stats.hits_by_source == {
CacheHitSource.HOST: 4,
CacheHitSource.DISK: 6,
CacheHitSource.P2P: 5,
}
assert sum(stats.hits_by_source.values()) == stats.hits == 15
def test_prefix_cache_stats_record_hits_by_source():
stats = PrefixCacheStats()
stats.record(num_tokens=32, num_hits=16, preempted=False)
stats.record(
num_tokens=32,
num_hits=24,
preempted=False,
hits_by_source={CacheHitSource.HOST: 16, CacheHitSource.DISK: 8},
)
# Preempted re-admissions never contribute to hits or the split.
stats.record(
num_tokens=32,
num_hits=32,
preempted=True,
hits_by_source={CacheHitSource.P2P: 32},
)
# A connector miscount is reported as unspecified, not dropped or raised.
stats.record(
num_tokens=8,
num_hits=8,
preempted=False,
hits_by_source={CacheHitSource.HOST: 4},
)
assert stats.requests == 3
assert stats.hits == 48
assert stats.preempted_hits == 32
assert stats.hits_by_source == {
CacheHitSource.HOST: 16,
CacheHitSource.DISK: 8,
CacheHitSource.EXTERNAL_UNSPECIFIED: 8,
}
assert sum(stats.hits_by_source.values()) == 48 - 16
def test_prefix_cache_stats_hits_by_source_serialization():
connector_stats = PrefixCacheStats()
connector_stats.record(
num_tokens=16,
num_hits=8,
preempted=False,
hits_by_source={CacheHitSource.P2P: 4, CacheHitSource.HOST: 4},
)
outputs = EngineCoreOutputs(
scheduler_stats=SchedulerStats(
connector_prefix_cache_stats=connector_stats,
)
)
encoded = MsgpackEncoder().encode(outputs)
decoded = MsgpackDecoder(EngineCoreOutputs).decode(encoded)
assert decoded.scheduler_stats is not None
assert decoded.scheduler_stats.connector_prefix_cache_stats is not None
assert decoded.scheduler_stats.connector_prefix_cache_stats.hits_by_source == {
CacheHitSource.P2P: 4,
CacheHitSource.HOST: 4,
}
def test_compute_iteration_details_includes_encoder_stats(): def test_compute_iteration_details_includes_encoder_stats():
scheduler_output = SchedulerOutput.make_empty() scheduler_output = SchedulerOutput.make_empty()
scheduler_output.scheduled_encoder_input_stats = ScheduledEncoderInputStats( scheduler_output.scheduled_encoder_input_stats = ScheduledEncoderInputStats(
@@ -23,6 +23,7 @@ from vllm.config import (
) )
from vllm.config.cache import MambaCacheMode from vllm.config.cache import MambaCacheMode
from vllm.config.kv_events import KVEventsConfig from vllm.config.kv_events import KVEventsConfig
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import ( from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
SimpleCPUOffloadConnector, SimpleCPUOffloadConnector,
) )
@@ -53,6 +54,7 @@ from vllm.v1.kv_cache_interface import (
MambaSpec, MambaSpec,
SlidingWindowSpec, SlidingWindowSpec,
) )
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request from vllm.v1.request import Request
from vllm.v1.simple_kv_offload.manager import SimpleCPUOffloadScheduler from vllm.v1.simple_kv_offload.manager import SimpleCPUOffloadScheduler
@@ -213,6 +215,37 @@ class SchedulerFixture:
num_groups: int = 1 num_groups: int = 1
@pytest.mark.parametrize(
("backend", "source"),
[("cpu", CacheHitSource.HOST), ("disk", CacheHitSource.DISK)],
)
def test_connector_reports_configured_cache_source(
tmp_path, backend: str, source: CacheHitSource
):
kv_cache_config = _make_kv_cache_config(num_blocks=4)
vllm_config = _make_vllm_config()
extra_config = {
"cpu_bytes_to_use_per_rank": _BYTES_PER_BLOCK * 4,
"kv_offload_backend": backend,
}
if backend == "disk":
extra_config.update(
disk_path=str(tmp_path / "kv-offload.bin"),
disk_capacity_bytes=_BYTES_PER_BLOCK * 4,
)
assert vllm_config.kv_transfer_config is not None
vllm_config.kv_transfer_config.kv_connector_extra_config = extra_config
connector = SimpleCPUOffloadConnector(
vllm_config,
KVConnectorRole.SCHEDULER,
kv_cache_config,
)
assert connector.get_external_cache_hit_sources(None, 32) == {source: 32} # type: ignore[arg-type]
assert connector.get_external_cache_hit_sources(None, 0) == {} # type: ignore[arg-type]
def make_scheduler( def make_scheduler(
num_cpu_blocks: int = 8, num_cpu_blocks: int = 8,
num_gpu_blocks: int = 16, num_gpu_blocks: int = 16,
+1
View File
@@ -114,6 +114,7 @@ def test_torch_profiler_records_each_profile_round(tmp_path):
profiler="torch", profiler="torch",
torch_profiler_dir=str(tmp_path), torch_profiler_dir=str(tmp_path),
torch_profiler_dump_cuda_time_total=False, torch_profiler_dump_cuda_time_total=False,
torch_profiler_with_stack=False,
), ),
worker_name="worker", worker_name="worker",
local_rank=1, local_rank=1,
@@ -51,6 +51,7 @@ import torch
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -179,6 +180,10 @@ class KVConnectorWorkerMetadata(ABC):
class KVConnectorBase_V1(ABC): class KVConnectorBase_V1(ABC):
"""Base class for KV connectors.""" """Base class for KV connectors."""
# Source label for this connector's external hits. Subclasses must set
# HOST/DISK/P2P or override get_external_cache_hit_sources().
_cache_hit_source = CacheHitSource.EXTERNAL_UNSPECIFIED
@property @property
def supports_divergent_local_hybrid_hits(self) -> bool: def supports_divergent_local_hybrid_hits(self) -> bool:
"""Whether external hits can complete divergent local hybrid hits. """Whether external hits can complete divergent local hybrid hits.
@@ -527,6 +532,24 @@ class KVConnectorBase_V1(ABC):
""" """
pass pass
def get_external_cache_hit_sources(
self,
request: "Request",
num_external_tokens: int,
) -> dict[CacheHitSource, int]:
"""Split ``num_external_tokens`` by the cache tier that supplied them.
Called after :meth:`update_state_after_alloc`, so the load plan is
known. Counts must sum to ``num_external_tokens``; a mismatch is
reported as ``external_unspecified``. Only non-zero counts are
included.
Default: all tokens under ``_cache_hit_source``.
"""
if num_external_tokens == 0:
return {}
return {self._cache_hit_source: num_external_tokens}
@abstractmethod @abstractmethod
def update_state_after_alloc( def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
@@ -67,6 +67,7 @@ from vllm.v1.kv_cache_interface import (
SlidingWindowMLASpec, SlidingWindowMLASpec,
SlidingWindowSpec, SlidingWindowSpec,
) )
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.request import RequestStatus from vllm.v1.request import RequestStatus
from vllm.v1.worker.block_table import BlockTable from vllm.v1.worker.block_table import BlockTable
from vllm.v1.worker.utils import select_common_block_size from vllm.v1.worker.utils import select_common_block_size
@@ -700,6 +701,8 @@ class MooncakeConnectorMetadata(KVConnectorMetadata):
class MooncakeConnector(KVConnectorBase_V1, SupportsHMA): class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
_cache_hit_source = CacheHitSource.P2P
def __init__( def __init__(
self, self,
vllm_config: VllmConfig, vllm_config: VllmConfig,
@@ -99,6 +99,7 @@ from vllm.v1.kv_cache_interface import (
SlidingWindowSpec, SlidingWindowSpec,
is_full_attention_spec, is_full_attention_spec,
) )
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import RequestStatus from vllm.v1.request import RequestStatus
@@ -222,6 +223,8 @@ def resolve_moriio_transfer_ack(
class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA): class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
_cache_hit_source = CacheHitSource.P2P
@property @property
def supports_divergent_local_hybrid_hits(self) -> bool: def supports_divergent_local_hybrid_hits(self) -> bool:
# The READ path always transfers the recurrent-state slot, including # The READ path always transfers the recurrent-state slot, including
@@ -31,6 +31,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -451,6 +452,18 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
to_return = (toks, load_async) to_return = (toks, load_async)
return to_return return to_return
def get_external_cache_hit_sources(
self,
request: "Request",
num_external_tokens: int,
) -> dict[CacheHitSource, int]:
chosen_connector = self._requests_to_connector.get(request.request_id)
if chosen_connector is None:
return super().get_external_cache_hit_sources(request, num_external_tokens)
return self._connectors[chosen_connector].get_external_cache_hit_sources(
request, num_external_tokens
)
def update_state_after_alloc( def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
): ):
@@ -58,6 +58,7 @@ from vllm.forward_context import ForwardContext
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -77,6 +78,8 @@ logger = init_logger(__name__)
class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA): class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
"""Base connector with common logic shared by pull and push modes.""" """Base connector with common logic shared by pull and push modes."""
_cache_hit_source = CacheHitSource.P2P
@property @property
def supports_divergent_local_hybrid_hits(self) -> bool: def supports_divergent_local_hybrid_hits(self) -> bool:
return True return True
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time import time
from collections import Counter
from collections.abc import Iterable, Sequence from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field, replace from dataclasses import dataclass, field, replace
from itertools import chain, islice from itertools import chain, islice
@@ -56,6 +57,7 @@ from vllm.v1.kv_offload.base import (
TierMatcher, TierMatcher,
make_offload_key, make_offload_key,
) )
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request, RequestStatus from vllm.v1.request import Request, RequestStatus
@@ -375,6 +377,9 @@ class RequestOffloadState:
# Fine-grained token boundary selected beyond the last complete offload # Fine-grained token boundary selected beyond the last complete offload
# chunk. It is consumed when the corresponding load is scheduled. # chunk. It is consumed when the corresponding load is scheduled.
partial_tail_boundary: int | None = None partial_tail_boundary: int | None = None
# Loaded attention keys cover their token positions. Recurrent state
# supports the whole reused external prefix.
load_key_ranges: list[tuple[int, int, OffloadKey]] = field(default_factory=list)
# True once on_request_finished has been signaled to the manager. # True once on_request_finished has been signaled to the manager.
finished_signaled: bool = False finished_signaled: bool = False
@@ -587,8 +592,9 @@ class OffloadingConnectorScheduler:
sliding_window_groups.sort(key=_sliding_window_sort_key, reverse=True) sliding_window_groups.sort(key=_sliding_window_sort_key, reverse=True)
# used by _lookup # used by _lookup
self._full_attention_groups: tuple[int, ...] = tuple(full_attention_groups)
self._sliding_window_groups: tuple[int, ...] = tuple(sliding_window_groups) self._sliding_window_groups: tuple[int, ...] = tuple(sliding_window_groups)
self._lookup_groups = tuple(full_attention_groups) + self._sliding_window_groups self._lookup_groups = self._full_attention_groups + self._sliding_window_groups
self._mamba_align_size: int | None = resolve_mamba_align_size( self._mamba_align_size: int | None = resolve_mamba_align_size(
spec, kv_cache_config spec, kv_cache_config
) )
@@ -1090,6 +1096,53 @@ class OffloadingConnectorScheduler:
return num_hit_tokens, bool(num_hit_tokens) return num_hit_tokens, bool(num_hit_tokens)
def get_external_cache_hit_sources(
self,
request: Request,
num_external_tokens: int,
) -> dict[CacheHitSource, int]:
"""Split the accepted external hit by the tier each loaded key came from.
Attention keys cover only their loaded token ranges; recurrent state
covers the whole reused prefix. Overlaps report the outermost tier.
Uncovered tokens, including omitted sliding-window history, are
``external_unspecified``.
"""
sources: dict[CacheHitSource, int] = {}
if num_external_tokens == 0:
return sources
req_status = self._req_status[request.request_id]
start = req_status.num_locally_computed_tokens
end = start + num_external_tokens
events: list[tuple[int, int, CacheHitSource]] = [
(end, 0, CacheHitSource.EXTERNAL_UNSPECIFIED)
]
for lo, hi, key in req_status.load_key_ranges:
lo, hi = max(start, lo), min(end, hi)
if lo >= hi:
continue
source = CacheHitSource(
self.manager.get_load_source(key, req_status.req_context)
)
events.extend(((lo, 1, source), (hi, -1, source)))
# Sweep endpoints instead of rescanning every range for each interval.
active: Counter[CacheHitSource] = Counter()
position = start
for boundary, delta, source in sorted(events):
if boundary > position:
tiers = [tier for tier, count in active.items() if count > 0]
tier = (
CacheHitSource.outermost(tiers)
if tiers
else CacheHitSource.EXTERNAL_UNSPECIFIED
)
sources[tier] = sources.get(tier, 0) + boundary - position
active[source] += delta
position = boundary
return sources
def update_state_after_alloc( def update_state_after_alloc(
self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int
): ):
@@ -1105,6 +1158,7 @@ class OffloadingConnectorScheduler:
assert partial_tail_boundary == num_cached_tokens assert partial_tail_boundary == num_cached_tokens
keys_to_load: list[OffloadKey] = [] keys_to_load: list[OffloadKey] = []
req_status.load_key_ranges.clear()
dst_block_ids: list[int] = [] dst_block_ids: list[int] = []
# per group # per group
group_sizes: list[int] = [] group_sizes: list[int] = []
@@ -1157,9 +1211,9 @@ class OffloadingConnectorScheduler:
) )
end_chunk_idx = num_chunks - (partial_tail_boundary is not None) end_chunk_idx = num_chunks - (partial_tail_boundary is not None)
assert len(offload_keys) >= end_chunk_idx assert len(offload_keys) >= end_chunk_idx
keys_to_load.extend(offload_keys[start_chunk_idx:end_chunk_idx]) group_keys_to_load = offload_keys[start_chunk_idx:end_chunk_idx]
if partial_tail_boundary is not None: if partial_tail_boundary is not None:
keys_to_load.append( group_keys_to_load.append(
self._make_boundary_key( self._make_boundary_key(
request, request,
group_config.group_idx, group_config.group_idx,
@@ -1167,6 +1221,24 @@ class OffloadingConnectorScheduler:
req_status.req_context, req_status.req_context,
) )
) )
keys_to_load.extend(group_keys_to_load)
if isinstance(group_config.kv_cache_spec, MambaSpec):
req_status.load_key_ranges.extend(
(num_locally_computed_tokens, num_cached_tokens, key)
for key in group_keys_to_load
)
else:
for chunk_idx, key in enumerate(
group_keys_to_load, start_chunk_idx
):
range_start = max(
load_start_gpu_block_idx * tokens_per_block,
chunk_idx * tokens_per_chunk,
)
range_end = min(
num_cached_tokens, (chunk_idx + 1) * tokens_per_chunk
)
req_status.load_key_ranges.append((range_start, range_end, key))
dst_block_ids.extend( dst_block_ids.extend(
block.block_id block.block_id
@@ -45,6 +45,7 @@ from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_offload.factory import OffloadingSpecFactory from vllm.v1.kv_offload.factory import OffloadingSpecFactory
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request from vllm.v1.request import Request
@@ -179,6 +180,16 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
bound = min(per_group_hits[group_id] for group_id in self._bounding_group_ids) bound = min(per_group_hits[group_id] for group_id in self._bounding_group_ids)
return max(0, bound - num_computed_tokens) return max(0, bound - num_computed_tokens)
def get_external_cache_hit_sources(
self,
request: "Request",
num_external_tokens: int,
) -> dict[CacheHitSource, int]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_external_cache_hit_sources(
request, num_external_tokens
)
def update_state_after_alloc( def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
): ):
@@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
) )
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.simple_kv_offload.manager import ( from vllm.v1.simple_kv_offload.manager import (
BoundaryStoreStats, BoundaryStoreStats,
@@ -109,6 +110,9 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
f"expected one of {VALID_KV_OFFLOAD_BACKENDS}" f"expected one of {VALID_KV_OFFLOAD_BACKENDS}"
) )
disk_mode = kv_offload_backend == "disk" disk_mode = kv_offload_backend == "disk"
self._cache_hit_source = (
CacheHitSource.DISK if disk_mode else CacheHitSource.HOST
)
disk_path = extra_config.get("disk_path", None) or None disk_path = extra_config.get("disk_path", None) or None
disk_capacity_bytes = int( disk_capacity_bytes = int(
+11 -1
View File
@@ -1261,10 +1261,20 @@ class Scheduler(SchedulerInterface):
self.connector_prefix_cache_stats is not None self.connector_prefix_cache_stats is not None
and connector_prefix_cache_queries != 0 and connector_prefix_cache_queries != 0
): ):
preempted = request.num_preemptions > 0
self.connector_prefix_cache_stats.record( self.connector_prefix_cache_stats.record(
num_tokens=connector_prefix_cache_queries, num_tokens=connector_prefix_cache_queries,
num_hits=connector_prefix_cache_hits, num_hits=connector_prefix_cache_hits,
preempted=request.num_preemptions > 0, preempted=preempted,
# The connector knows the tier only after the load
# plan above exists.
hits_by_source=(
self.connector.get_external_cache_hit_sources(
request, connector_prefix_cache_hits
)
if connector_prefix_cache_hits and not preempted
else None
),
) )
# Record at admission so unscheduled lookups are not counted. # Record at admission so unscheduled lookups are not counted.
+13
View File
@@ -18,6 +18,7 @@ if TYPE_CHECKING:
from vllm.v1.kv_hints import KvHintsEnvelope from vllm.v1.kv_hints import KvHintsEnvelope
from vllm.v1.kv_offload.config import OffloadingConfig from vllm.v1.kv_offload.config import OffloadingConfig
from vllm.v1.metrics.cache_hit_source import CacheHitSource
# `OffloadKey` identifies an offloaded block. It combines a block hash with # `OffloadKey` identifies an offloaded block. It combines a block hash with
# its KV cache group index, encoded as raw bytes to avoid tuple GC overhead. # its KV cache group index, encoded as raw bytes to avoid tuple GC overhead.
@@ -248,6 +249,18 @@ class OffloadingManager(ABC):
""" """
pass pass
def get_load_source(
self, key: OffloadKey, req_context: ReqContext
) -> CacheHitSource:
"""Return the cache tier that supplied a successful lookup.
This is queried only after ``lookup`` resolves to ``HIT``. Composing
managers can override it to retain the origin across asynchronous
promotion into a primary tier. The default keeps out-of-tree managers
compatible while making unknown provenance explicit.
"""
return CacheHitSource.EXTERNAL_UNSPECIFIED
@abstractmethod @abstractmethod
def prepare_load( def prepare_load(
self, self,
+7
View File
@@ -27,6 +27,7 @@ from vllm.v1.kv_offload.cpu.common import (
) )
from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus from vllm.v1.kv_offload.cpu.policies.base import CachePolicy, ChunkStatus
from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
from vllm.v1.metrics.cache_hit_source import CacheHitSource
@dataclass(slots=True) @dataclass(slots=True)
@@ -193,6 +194,12 @@ class CPUOffloadingManager(OffloadingManager):
return LookupResult.HIT_PENDING return LookupResult.HIT_PENDING
return LookupResult.HIT return LookupResult.HIT
@override
def get_load_source(
self, key: OffloadKey, req_context: ReqContext
) -> CacheHitSource:
return CacheHitSource.HOST
@override @override
def prepare_load( def prepare_load(
self, self,
+12
View File
@@ -24,6 +24,7 @@ from vllm.v1.kv_offload.base import (
from vllm.v1.kv_offload.tiering.backpressure import ( from vllm.v1.kv_offload.tiering.backpressure import (
BackpressureDetector, BackpressureDetector,
) )
from vllm.v1.metrics.cache_hit_source import CacheHitSource
if TYPE_CHECKING: if TYPE_CHECKING:
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
@@ -141,6 +142,17 @@ class SecondaryTierManager(ABC):
""" """
medium: ClassVar[Medium | None] = None medium: ClassVar[Medium | None] = None
# Cache tier reported for hits served from this tier. Derived from
# ``medium`` unless a subclass sets it; ``None`` medium means unknown.
cache_hit_source: ClassVar[CacheHitSource] = CacheHitSource.EXTERNAL_UNSPECIFIED
def __init_subclass__(cls, **kwargs: Any) -> None:
super().__init_subclass__(**kwargs)
if "cache_hit_source" not in cls.__dict__ and cls.medium is not None:
cls.cache_hit_source = {
Medium.CPU: CacheHitSource.HOST,
Medium.STORAGE: CacheHitSource.DISK,
}[cls.medium]
def __init__( def __init__(
self, self,
+28
View File
@@ -54,6 +54,7 @@ from vllm.v1.kv_offload.tiering.base import (
TransferJob, TransferJob,
) )
from vllm.v1.kv_offload.tiering.metrics import TieringMetricsTracker from vllm.v1.kv_offload.tiering.metrics import TieringMetricsTracker
from vllm.v1.metrics.cache_hit_source import CacheHitSource
logger = init_logger(__name__) logger = init_logger(__name__)
@@ -232,6 +233,11 @@ class TieringOffloadingManager(OffloadingManager):
# complete_store(), since complete_store() can still submit cascades. # complete_store(), since complete_store() can still submit cascades.
self._req_state: dict[str, RequestState] = {} self._req_state: dict[str, RequestState] = {}
# Preserve the original tier for this request's cache-hit metrics,
# even after its KV blocks are promoted into host memory
# (otherwise a disk or P2P hit would be mislabeled as a host hit).
self._request_load_sources: dict[str, dict[OffloadKey, CacheHitSource]] = {}
# Cached ParentManager wrappers for each secondary tier. # Cached ParentManager wrappers for each secondary tier.
self._tier_parents: dict[SecondaryTierManager, _SecondaryTierFacingParent] = { self._tier_parents: dict[SecondaryTierManager, _SecondaryTierFacingParent] = {
tier: _SecondaryTierFacingParent(self, tier_idx) tier: _SecondaryTierFacingParent(self, tier_idx)
@@ -293,6 +299,13 @@ class TieringOffloadingManager(OffloadingManager):
successful_keys = () successful_keys = ()
failed_keys = transfer_job.keys failed_keys = transfer_job.keys
load_sources = self._request_load_sources.get(transfer_job.req_context.req_id)
if load_sources is not None:
source = self.secondary_tiers[job_metadata.tier_idx].cache_hit_source
for key in failed_keys:
if load_sources.get(key) == source:
del load_sources[key]
if successful_keys: if successful_keys:
self.primary_tier.complete_write( self.primary_tier.complete_write(
successful_keys, successful_keys,
@@ -464,6 +477,15 @@ class TieringOffloadingManager(OffloadingManager):
return LookupResult.RETRY return LookupResult.RETRY
return LookupResult.MISS return LookupResult.MISS
@override
def get_load_source(
self, key: OffloadKey, req_context: ReqContext
) -> CacheHitSource:
load_sources = self._request_load_sources.get(req_context.req_id)
if load_sources is not None and key in load_sources:
return load_sources[key]
return self.primary_tier.get_load_source(key, req_context)
def _initiate_promotion( def _initiate_promotion(
self, self,
tier_idx: int, tier_idx: int,
@@ -501,6 +523,10 @@ class TieringOffloadingManager(OffloadingManager):
store_spec = primary_write_result.store_spec store_spec = primary_write_result.store_spec
assert isinstance(store_spec, CPULoadStoreSpec) assert isinstance(store_spec, CPULoadStoreSpec)
load_sources = self._request_load_sources.setdefault(req_context.req_id, {})
source = self.secondary_tiers[tier_idx].cache_hit_source
for promoted_key in primary_write_result.keys_to_store:
load_sources[promoted_key] = source
# Defer submit_load to on_schedule_end(). Group by (tier, request) so # Defer submit_load to on_schedule_end(). Group by (tier, request) so
# each request's chunks are submitted as one batched job per tier. # each request's chunks are submitted as one batched job per tier.
tier_pending = self._pending_load_submissions.setdefault(tier_idx, {}) tier_pending = self._pending_load_submissions.setdefault(tier_idx, {})
@@ -848,6 +874,7 @@ class TieringOffloadingManager(OffloadingManager):
continue continue
tier.on_request_finished(state.req_context) tier.on_request_finished(state.req_context)
self._metrics.on_request_finished(state.req_context) self._metrics.on_request_finished(state.req_context)
self._request_load_sources.pop(req_id, None)
del self._req_state[req_id] del self._req_state[req_id]
@override @override
@@ -930,6 +957,7 @@ class TieringOffloadingManager(OffloadingManager):
# reset below invalidates; their submit_load() has not yet been # reset below invalidates; their submit_load() has not yet been
# called so no tier I/O is touching that memory. # called so no tier I/O is touching that memory.
self._pending_load_submissions.clear() self._pending_load_submissions.clear()
self._request_load_sources.clear()
self._metrics.assert_idle() self._metrics.assert_idle()
finished_req_ids = [] finished_req_ids = []
+4 -1
View File
@@ -11,7 +11,7 @@ import time
import uuid import uuid
from collections.abc import Iterable, Sequence from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any, ClassVar
from typing_extensions import override from typing_extensions import override
@@ -35,6 +35,7 @@ from vllm.v1.kv_offload.tiering.base import (
from vllm.v1.kv_offload.tiering.p2p.control import ControlTransport, ZmqTransport from vllm.v1.kv_offload.tiering.p2p.control import ControlTransport, ZmqTransport
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport, NixlTransport from vllm.v1.kv_offload.tiering.p2p.data import DataTransport, NixlTransport
from vllm.v1.kv_offload.tiering.p2p.session import P2PSession from vllm.v1.kv_offload.tiering.p2p.session import P2PSession
from vllm.v1.metrics.cache_hit_source import CacheHitSource
if TYPE_CHECKING: if TYPE_CHECKING:
from vllm.v1.kv_offload.base import OffloadingSpec from vllm.v1.kv_offload.base import OffloadingSpec
@@ -206,6 +207,8 @@ class P2PSecondaryTierManager(SecondaryTierManager):
and existing sessions are polled even when no requests are scheduled. and existing sessions are polled even when no requests are scheduled.
""" """
cache_hit_source: ClassVar[CacheHitSource] = CacheHitSource.P2P
def __init__( def __init__(
self, self,
offloading_spec: OffloadingSpec, offloading_spec: OffloadingSpec,
+32
View File
@@ -0,0 +1,32 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from collections.abc import Iterable
from enum import Enum
class CacheHitSource(str, Enum):
"""Where cached prompt tokens' KV came from.
Members are listed in the order blocks are offloaded: accelerator, then
host memory, then secondary tiers. Per-source token counts are plain
``dict[CacheHitSource, int]`` mappings holding only non-zero entries.
"""
DEVICE = "device"
HOST = "host"
P2P = "p2p"
DISK = "disk"
# Fallback: uninstrumented connector or remote store. Last, so mixing
# with a known tier never reports the known one.
EXTERNAL_UNSPECIFIED = "external_unspecified"
@classmethod
def outermost(cls, sources: Iterable[CacheHitSource]) -> CacheHitSource:
"""The tier in ``sources`` farthest from the accelerator."""
return max(sources, key=_SOURCE_ORDER.__getitem__)
_SOURCE_ORDER = {source: rank for rank, source in enumerate(CacheHitSource)}
+31 -3
View File
@@ -23,6 +23,7 @@ from vllm.logger import init_logger
from vllm.plugins import STAT_LOGGER_PLUGINS_GROUP, load_plugins_by_group from vllm.plugins import STAT_LOGGER_PLUGINS_GROUP, load_plugins_by_group
from vllm.v1.engine import FinishReason from vllm.v1.engine import FinishReason
from vllm.v1.metrics.buckets import histogram_buckets from vllm.v1.metrics.buckets import histogram_buckets
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.metrics.perf import PerfMetricsLogging, PerfMetricsProm from vllm.v1.metrics.perf import PerfMetricsLogging, PerfMetricsProm
from vllm.v1.metrics.prometheus import unregister_vllm_metrics from vllm.v1.metrics.prometheus import unregister_vllm_metrics
from vllm.v1.metrics.stats import ( from vllm.v1.metrics.stats import (
@@ -752,6 +753,23 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
counter_prompt_tokens_cached, per_engine_labelvalues counter_prompt_tokens_cached, per_engine_labelvalues
) )
self.counter_prompt_tokens_cached_by_source = self._counter_cls(
name="vllm:prompt_tokens_cached_by_source",
documentation=(
"Prefix-cache hit tokens by the cache tier that supplied them: "
"device (local HBM), host (offloaded to DRAM), disk, p2p "
"(transferred from another vLLM instance) or "
"external_unspecified. Counted at admission; sums to "
"vllm:prefix_cache_hits + vllm:external_prefix_cache_hits."
),
labelnames=labelnames + ["source"],
)
for source in CacheHitSource:
for labelvalues in per_engine_labelvalues.values():
self.counter_prompt_tokens_cached_by_source.labels(
*labelvalues, source.value
)
counter_generation_tokens = self._counter_cls( counter_generation_tokens = self._counter_cls(
name="vllm:generation_tokens", name="vllm:generation_tokens",
documentation="Number of generation tokens processed.", documentation="Number of generation tokens processed.",
@@ -1111,13 +1129,23 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
scheduler_stats.prefix_cache_stats.hits scheduler_stats.prefix_cache_stats.hits
) )
if scheduler_stats.connector_prefix_cache_stats is not None: labelvalues = self.per_engine_labelvalues[engine_idx]
self.counter_prompt_tokens_cached_by_source.labels(
*labelvalues, CacheHitSource.DEVICE.value
).inc(scheduler_stats.prefix_cache_stats.hits)
connector_stats = scheduler_stats.connector_prefix_cache_stats
if connector_stats is not None:
self.counter_connector_prefix_cache_queries[engine_idx].inc( self.counter_connector_prefix_cache_queries[engine_idx].inc(
scheduler_stats.connector_prefix_cache_stats.queries connector_stats.queries
) )
self.counter_connector_prefix_cache_hits[engine_idx].inc( self.counter_connector_prefix_cache_hits[engine_idx].inc(
scheduler_stats.connector_prefix_cache_stats.hits connector_stats.hits
) )
for tier, num_tokens in connector_stats.hits_by_source.items():
self.counter_prompt_tokens_cached_by_source.labels(
*labelvalues, tier.value
).inc(num_tokens)
if scheduler_stats.spec_decoding_stats is not None: if scheduler_stats.spec_decoding_stats is not None:
self.spec_decoding_prom.observe( self.spec_decoding_prom.observe(
+33 -6
View File
@@ -8,9 +8,13 @@ from typing import TYPE_CHECKING, Any
import vllm.envs as envs import vllm.envs as envs
from vllm.compilation.cuda_graph import CUDAGraphStat from vllm.compilation.cuda_graph import CUDAGraphStat
from vllm.logger import init_logger
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.metrics.perf import PerfStats from vllm.v1.metrics.perf import PerfStats
from vllm.v1.spec_decode.metrics import SpecDecodingStats from vllm.v1.spec_decode.metrics import SpecDecodingStats
logger = init_logger(__name__)
if TYPE_CHECKING: if TYPE_CHECKING:
from vllm.v1.engine import EngineCoreEvent, EngineCoreOutput, FinishReason from vllm.v1.engine import EngineCoreEvent, EngineCoreOutput, FinishReason
@@ -128,19 +132,42 @@ class PrefixCacheStats(BaseCacheStats):
preempted_hits: int = 0 preempted_hits: int = 0
"""The `hits` number for preempted requests.""" """The `hits` number for preempted requests."""
hits_by_source: dict[CacheHitSource, int] = field(default_factory=dict)
"""`hits` split by the cache tier that supplied them (connector stats only)."""
def record(self, num_tokens: int, num_hits: int, preempted: bool) -> None: def record(
self,
num_tokens: int,
num_hits: int,
preempted: bool,
hits_by_source: dict[CacheHitSource, int] | None = None,
) -> None:
"""Aggregate request information into the stats.""" """Aggregate request information into the stats."""
if preempted: if preempted:
# Previously preempted request # Previously preempted request
self.preempted_requests += 1 self.preempted_requests += 1
self.preempted_queries += num_tokens self.preempted_queries += num_tokens
self.preempted_hits += num_hits self.preempted_hits += num_hits
else: return
# New request # New request
self.requests += 1 self.requests += 1
self.queries += num_tokens self.queries += num_tokens
self.hits += num_hits self.hits += num_hits
if hits_by_source is None:
return
if sum(hits_by_source.values()) != num_hits:
logger.warning_once(
"Connector attributed %s for %d cached tokens; reporting them as %s.",
str(hits_by_source),
num_hits,
CacheHitSource.EXTERNAL_UNSPECIFIED.value,
)
hits_by_source = {CacheHitSource.EXTERNAL_UNSPECIFIED: num_hits}
for source, num_source_hits in hits_by_source.items():
if num_source_hits:
self.hits_by_source[source] = (
self.hits_by_source.get(source, 0) + num_source_hits
)
@dataclass @dataclass