[Metrics] Expose cached prompt tokens by cache tier (#56318)

Signed-off-by: Cam Quilici <cjquilici@gmail.com>
Signed-off-by: Cam Quilici <cameron@semianalysis.com>
Co-authored-by: Cam Quilici <cameron@semianalysis.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Cameron Quilici
2026-10-06 14:52:05 -04:00
committed by GitHub
co-authored by Cam Quilici Nick Hill Yifan Qiao
parent ad0f67a7ec
commit e37e51dd24
36 changed files with 1005 additions and 44 deletions
@@ -51,6 +51,7 @@ import torch
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
@@ -179,6 +180,10 @@ class KVConnectorWorkerMetadata(ABC):
class KVConnectorBase_V1(ABC):
"""Base class for KV connectors."""
# Source label for this connector's external hits. Subclasses must set
# HOST/DISK/P2P or override get_external_cache_hit_sources().
_cache_hit_source = CacheHitSource.EXTERNAL_UNSPECIFIED
@property
def supports_divergent_local_hybrid_hits(self) -> bool:
"""Whether external hits can complete divergent local hybrid hits.
@@ -527,6 +532,24 @@ class KVConnectorBase_V1(ABC):
"""
pass
def get_external_cache_hit_sources(
self,
request: "Request",
num_external_tokens: int,
) -> dict[CacheHitSource, int]:
"""Split ``num_external_tokens`` by the cache tier that supplied them.
Called after :meth:`update_state_after_alloc`, so the load plan is
known. Counts must sum to ``num_external_tokens``; a mismatch is
reported as ``external_unspecified``. Only non-zero counts are
included.
Default: all tokens under ``_cache_hit_source``.
"""
if num_external_tokens == 0:
return {}
return {self._cache_hit_source: num_external_tokens}
@abstractmethod
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
@@ -67,6 +67,7 @@ from vllm.v1.kv_cache_interface import (
SlidingWindowMLASpec,
SlidingWindowSpec,
)
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.request import RequestStatus
from vllm.v1.worker.block_table import BlockTable
from vllm.v1.worker.utils import select_common_block_size
@@ -700,6 +701,8 @@ class MooncakeConnectorMetadata(KVConnectorMetadata):
class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
_cache_hit_source = CacheHitSource.P2P
def __init__(
self,
vllm_config: VllmConfig,
@@ -99,6 +99,7 @@ from vllm.v1.kv_cache_interface import (
SlidingWindowSpec,
is_full_attention_spec,
)
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import RequestStatus
@@ -222,6 +223,8 @@ def resolve_moriio_transfer_ack(
class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
_cache_hit_source = CacheHitSource.P2P
@property
def supports_divergent_local_hybrid_hits(self) -> bool:
# The READ path always transfers the recurrent-state slot, including
@@ -31,6 +31,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
@@ -451,6 +452,18 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
to_return = (toks, load_async)
return to_return
def get_external_cache_hit_sources(
self,
request: "Request",
num_external_tokens: int,
) -> dict[CacheHitSource, int]:
chosen_connector = self._requests_to_connector.get(request.request_id)
if chosen_connector is None:
return super().get_external_cache_hit_sources(request, num_external_tokens)
return self._connectors[chosen_connector].get_external_cache_hit_sources(
request, num_external_tokens
)
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
@@ -58,6 +58,7 @@ from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
@@ -77,6 +78,8 @@ logger = init_logger(__name__)
class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
"""Base connector with common logic shared by pull and push modes."""
_cache_hit_source = CacheHitSource.P2P
@property
def supports_divergent_local_hybrid_hits(self) -> bool:
return True
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections import Counter
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field, replace
from itertools import chain, islice
@@ -56,6 +57,7 @@ from vllm.v1.kv_offload.base import (
TierMatcher,
make_offload_key,
)
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request, RequestStatus
@@ -375,6 +377,9 @@ class RequestOffloadState:
# Fine-grained token boundary selected beyond the last complete offload
# chunk. It is consumed when the corresponding load is scheduled.
partial_tail_boundary: int | None = None
# Loaded attention keys cover their token positions. Recurrent state
# supports the whole reused external prefix.
load_key_ranges: list[tuple[int, int, OffloadKey]] = field(default_factory=list)
# True once on_request_finished has been signaled to the manager.
finished_signaled: bool = False
@@ -587,8 +592,9 @@ class OffloadingConnectorScheduler:
sliding_window_groups.sort(key=_sliding_window_sort_key, reverse=True)
# used by _lookup
self._full_attention_groups: tuple[int, ...] = tuple(full_attention_groups)
self._sliding_window_groups: tuple[int, ...] = tuple(sliding_window_groups)
self._lookup_groups = tuple(full_attention_groups) + self._sliding_window_groups
self._lookup_groups = self._full_attention_groups + self._sliding_window_groups
self._mamba_align_size: int | None = resolve_mamba_align_size(
spec, kv_cache_config
)
@@ -1090,6 +1096,53 @@ class OffloadingConnectorScheduler:
return num_hit_tokens, bool(num_hit_tokens)
def get_external_cache_hit_sources(
self,
request: Request,
num_external_tokens: int,
) -> dict[CacheHitSource, int]:
"""Split the accepted external hit by the tier each loaded key came from.
Attention keys cover only their loaded token ranges; recurrent state
covers the whole reused prefix. Overlaps report the outermost tier.
Uncovered tokens, including omitted sliding-window history, are
``external_unspecified``.
"""
sources: dict[CacheHitSource, int] = {}
if num_external_tokens == 0:
return sources
req_status = self._req_status[request.request_id]
start = req_status.num_locally_computed_tokens
end = start + num_external_tokens
events: list[tuple[int, int, CacheHitSource]] = [
(end, 0, CacheHitSource.EXTERNAL_UNSPECIFIED)
]
for lo, hi, key in req_status.load_key_ranges:
lo, hi = max(start, lo), min(end, hi)
if lo >= hi:
continue
source = CacheHitSource(
self.manager.get_load_source(key, req_status.req_context)
)
events.extend(((lo, 1, source), (hi, -1, source)))
# Sweep endpoints instead of rescanning every range for each interval.
active: Counter[CacheHitSource] = Counter()
position = start
for boundary, delta, source in sorted(events):
if boundary > position:
tiers = [tier for tier, count in active.items() if count > 0]
tier = (
CacheHitSource.outermost(tiers)
if tiers
else CacheHitSource.EXTERNAL_UNSPECIFIED
)
sources[tier] = sources.get(tier, 0) + boundary - position
active[source] += delta
position = boundary
return sources
def update_state_after_alloc(
self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int
):
@@ -1105,6 +1158,7 @@ class OffloadingConnectorScheduler:
assert partial_tail_boundary == num_cached_tokens
keys_to_load: list[OffloadKey] = []
req_status.load_key_ranges.clear()
dst_block_ids: list[int] = []
# per group
group_sizes: list[int] = []
@@ -1157,9 +1211,9 @@ class OffloadingConnectorScheduler:
)
end_chunk_idx = num_chunks - (partial_tail_boundary is not None)
assert len(offload_keys) >= end_chunk_idx
keys_to_load.extend(offload_keys[start_chunk_idx:end_chunk_idx])
group_keys_to_load = offload_keys[start_chunk_idx:end_chunk_idx]
if partial_tail_boundary is not None:
keys_to_load.append(
group_keys_to_load.append(
self._make_boundary_key(
request,
group_config.group_idx,
@@ -1167,6 +1221,24 @@ class OffloadingConnectorScheduler:
req_status.req_context,
)
)
keys_to_load.extend(group_keys_to_load)
if isinstance(group_config.kv_cache_spec, MambaSpec):
req_status.load_key_ranges.extend(
(num_locally_computed_tokens, num_cached_tokens, key)
for key in group_keys_to_load
)
else:
for chunk_idx, key in enumerate(
group_keys_to_load, start_chunk_idx
):
range_start = max(
load_start_gpu_block_idx * tokens_per_block,
chunk_idx * tokens_per_chunk,
)
range_end = min(
num_cached_tokens, (chunk_idx + 1) * tokens_per_chunk
)
req_status.load_key_ranges.append((range_start, range_end, key))
dst_block_ids.extend(
block.block_id
@@ -45,6 +45,7 @@ from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
@@ -179,6 +180,16 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
bound = min(per_group_hits[group_id] for group_id in self._bounding_group_ids)
return max(0, bound - num_computed_tokens)
def get_external_cache_hit_sources(
self,
request: "Request",
num_external_tokens: int,
) -> dict[CacheHitSource, int]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_external_cache_hit_sources(
request, num_external_tokens
)
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
@@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
)
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.metrics.cache_hit_source import CacheHitSource
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.simple_kv_offload.manager import (
BoundaryStoreStats,
@@ -109,6 +110,9 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
f"expected one of {VALID_KV_OFFLOAD_BACKENDS}"
)
disk_mode = kv_offload_backend == "disk"
self._cache_hit_source = (
CacheHitSource.DISK if disk_mode else CacheHitSource.HOST
)
disk_path = extra_config.get("disk_path", None) or None
disk_capacity_bytes = int(