[Metrics] Expose cached prompt tokens by cache tier (#56318)
Signed-off-by: Cam Quilici <cjquilici@gmail.com> Signed-off-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Cam Quilici
Nick Hill
Yifan Qiao
parent
ad0f67a7ec
commit
e37e51dd24
@@ -51,6 +51,7 @@ import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -179,6 +180,10 @@ class KVConnectorWorkerMetadata(ABC):
|
||||
class KVConnectorBase_V1(ABC):
|
||||
"""Base class for KV connectors."""
|
||||
|
||||
# Source label for this connector's external hits. Subclasses must set
|
||||
# HOST/DISK/P2P or override get_external_cache_hit_sources().
|
||||
_cache_hit_source = CacheHitSource.EXTERNAL_UNSPECIFIED
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
"""Whether external hits can complete divergent local hybrid hits.
|
||||
@@ -527,6 +532,24 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_external_cache_hit_sources(
|
||||
self,
|
||||
request: "Request",
|
||||
num_external_tokens: int,
|
||||
) -> dict[CacheHitSource, int]:
|
||||
"""Split ``num_external_tokens`` by the cache tier that supplied them.
|
||||
|
||||
Called after :meth:`update_state_after_alloc`, so the load plan is
|
||||
known. Counts must sum to ``num_external_tokens``; a mismatch is
|
||||
reported as ``external_unspecified``. Only non-zero counts are
|
||||
included.
|
||||
|
||||
Default: all tokens under ``_cache_hit_source``.
|
||||
"""
|
||||
if num_external_tokens == 0:
|
||||
return {}
|
||||
return {self._cache_hit_source: num_external_tokens}
|
||||
|
||||
@abstractmethod
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
|
||||
@@ -67,6 +67,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.request import RequestStatus
|
||||
from vllm.v1.worker.block_table import BlockTable
|
||||
from vllm.v1.worker.utils import select_common_block_size
|
||||
@@ -700,6 +701,8 @@ class MooncakeConnectorMetadata(KVConnectorMetadata):
|
||||
|
||||
|
||||
class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
_cache_hit_source = CacheHitSource.P2P
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
|
||||
@@ -99,6 +99,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
SlidingWindowSpec,
|
||||
is_full_attention_spec,
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
@@ -222,6 +223,8 @@ def resolve_moriio_transfer_ack(
|
||||
|
||||
|
||||
class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
_cache_hit_source = CacheHitSource.P2P
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
# The READ path always transfers the recurrent-state slot, including
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -451,6 +452,18 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
to_return = (toks, load_async)
|
||||
return to_return
|
||||
|
||||
def get_external_cache_hit_sources(
|
||||
self,
|
||||
request: "Request",
|
||||
num_external_tokens: int,
|
||||
) -> dict[CacheHitSource, int]:
|
||||
chosen_connector = self._requests_to_connector.get(request.request_id)
|
||||
if chosen_connector is None:
|
||||
return super().get_external_cache_hit_sources(request, num_external_tokens)
|
||||
return self._connectors[chosen_connector].get_external_cache_hit_sources(
|
||||
request, num_external_tokens
|
||||
)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
|
||||
@@ -58,6 +58,7 @@ from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -77,6 +78,8 @@ logger = init_logger(__name__)
|
||||
class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""Base connector with common logic shared by pull and push modes."""
|
||||
|
||||
_cache_hit_source = CacheHitSource.P2P
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
return True
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from itertools import chain, islice
|
||||
@@ -56,6 +57,7 @@ from vllm.v1.kv_offload.base import (
|
||||
TierMatcher,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
@@ -375,6 +377,9 @@ class RequestOffloadState:
|
||||
# Fine-grained token boundary selected beyond the last complete offload
|
||||
# chunk. It is consumed when the corresponding load is scheduled.
|
||||
partial_tail_boundary: int | None = None
|
||||
# Loaded attention keys cover their token positions. Recurrent state
|
||||
# supports the whole reused external prefix.
|
||||
load_key_ranges: list[tuple[int, int, OffloadKey]] = field(default_factory=list)
|
||||
# True once on_request_finished has been signaled to the manager.
|
||||
finished_signaled: bool = False
|
||||
|
||||
@@ -587,8 +592,9 @@ class OffloadingConnectorScheduler:
|
||||
sliding_window_groups.sort(key=_sliding_window_sort_key, reverse=True)
|
||||
|
||||
# used by _lookup
|
||||
self._full_attention_groups: tuple[int, ...] = tuple(full_attention_groups)
|
||||
self._sliding_window_groups: tuple[int, ...] = tuple(sliding_window_groups)
|
||||
self._lookup_groups = tuple(full_attention_groups) + self._sliding_window_groups
|
||||
self._lookup_groups = self._full_attention_groups + self._sliding_window_groups
|
||||
self._mamba_align_size: int | None = resolve_mamba_align_size(
|
||||
spec, kv_cache_config
|
||||
)
|
||||
@@ -1090,6 +1096,53 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
return num_hit_tokens, bool(num_hit_tokens)
|
||||
|
||||
def get_external_cache_hit_sources(
|
||||
self,
|
||||
request: Request,
|
||||
num_external_tokens: int,
|
||||
) -> dict[CacheHitSource, int]:
|
||||
"""Split the accepted external hit by the tier each loaded key came from.
|
||||
|
||||
Attention keys cover only their loaded token ranges; recurrent state
|
||||
covers the whole reused prefix. Overlaps report the outermost tier.
|
||||
Uncovered tokens, including omitted sliding-window history, are
|
||||
``external_unspecified``.
|
||||
"""
|
||||
sources: dict[CacheHitSource, int] = {}
|
||||
if num_external_tokens == 0:
|
||||
return sources
|
||||
req_status = self._req_status[request.request_id]
|
||||
start = req_status.num_locally_computed_tokens
|
||||
end = start + num_external_tokens
|
||||
|
||||
events: list[tuple[int, int, CacheHitSource]] = [
|
||||
(end, 0, CacheHitSource.EXTERNAL_UNSPECIFIED)
|
||||
]
|
||||
for lo, hi, key in req_status.load_key_ranges:
|
||||
lo, hi = max(start, lo), min(end, hi)
|
||||
if lo >= hi:
|
||||
continue
|
||||
source = CacheHitSource(
|
||||
self.manager.get_load_source(key, req_status.req_context)
|
||||
)
|
||||
events.extend(((lo, 1, source), (hi, -1, source)))
|
||||
|
||||
# Sweep endpoints instead of rescanning every range for each interval.
|
||||
active: Counter[CacheHitSource] = Counter()
|
||||
position = start
|
||||
for boundary, delta, source in sorted(events):
|
||||
if boundary > position:
|
||||
tiers = [tier for tier, count in active.items() if count > 0]
|
||||
tier = (
|
||||
CacheHitSource.outermost(tiers)
|
||||
if tiers
|
||||
else CacheHitSource.EXTERNAL_UNSPECIFIED
|
||||
)
|
||||
sources[tier] = sources.get(tier, 0) + boundary - position
|
||||
active[source] += delta
|
||||
position = boundary
|
||||
return sources
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int
|
||||
):
|
||||
@@ -1105,6 +1158,7 @@ class OffloadingConnectorScheduler:
|
||||
assert partial_tail_boundary == num_cached_tokens
|
||||
|
||||
keys_to_load: list[OffloadKey] = []
|
||||
req_status.load_key_ranges.clear()
|
||||
dst_block_ids: list[int] = []
|
||||
# per group
|
||||
group_sizes: list[int] = []
|
||||
@@ -1157,9 +1211,9 @@ class OffloadingConnectorScheduler:
|
||||
)
|
||||
end_chunk_idx = num_chunks - (partial_tail_boundary is not None)
|
||||
assert len(offload_keys) >= end_chunk_idx
|
||||
keys_to_load.extend(offload_keys[start_chunk_idx:end_chunk_idx])
|
||||
group_keys_to_load = offload_keys[start_chunk_idx:end_chunk_idx]
|
||||
if partial_tail_boundary is not None:
|
||||
keys_to_load.append(
|
||||
group_keys_to_load.append(
|
||||
self._make_boundary_key(
|
||||
request,
|
||||
group_config.group_idx,
|
||||
@@ -1167,6 +1221,24 @@ class OffloadingConnectorScheduler:
|
||||
req_status.req_context,
|
||||
)
|
||||
)
|
||||
keys_to_load.extend(group_keys_to_load)
|
||||
if isinstance(group_config.kv_cache_spec, MambaSpec):
|
||||
req_status.load_key_ranges.extend(
|
||||
(num_locally_computed_tokens, num_cached_tokens, key)
|
||||
for key in group_keys_to_load
|
||||
)
|
||||
else:
|
||||
for chunk_idx, key in enumerate(
|
||||
group_keys_to_load, start_chunk_idx
|
||||
):
|
||||
range_start = max(
|
||||
load_start_gpu_block_idx * tokens_per_block,
|
||||
chunk_idx * tokens_per_chunk,
|
||||
)
|
||||
range_end = min(
|
||||
num_cached_tokens, (chunk_idx + 1) * tokens_per_chunk
|
||||
)
|
||||
req_status.load_key_ranges.append((range_start, range_end, key))
|
||||
|
||||
dst_block_ids.extend(
|
||||
block.block_id
|
||||
|
||||
@@ -45,6 +45,7 @@ from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.kv_offload.factory import OffloadingSpecFactory
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
@@ -179,6 +180,16 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
bound = min(per_group_hits[group_id] for group_id in self._bounding_group_ids)
|
||||
return max(0, bound - num_computed_tokens)
|
||||
|
||||
def get_external_cache_hit_sources(
|
||||
self,
|
||||
request: "Request",
|
||||
num_external_tokens: int,
|
||||
) -> dict[CacheHitSource, int]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.get_external_cache_hit_sources(
|
||||
request, num_external_tokens
|
||||
)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.simple_kv_offload.manager import (
|
||||
BoundaryStoreStats,
|
||||
@@ -109,6 +110,9 @@ class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
f"expected one of {VALID_KV_OFFLOAD_BACKENDS}"
|
||||
)
|
||||
disk_mode = kv_offload_backend == "disk"
|
||||
self._cache_hit_source = (
|
||||
CacheHitSource.DISK if disk_mode else CacheHitSource.HOST
|
||||
)
|
||||
|
||||
disk_path = extra_config.get("disk_path", None) or None
|
||||
disk_capacity_bytes = int(
|
||||
|
||||
Reference in New Issue
Block a user