e37e51dd24
Signed-off-by: Cam Quilici <cjquilici@gmail.com> Signed-off-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
33 lines
1.0 KiB
Python
33 lines
1.0 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Iterable
|
|
from enum import Enum
|
|
|
|
|
|
class CacheHitSource(str, Enum):
|
|
"""Where cached prompt tokens' KV came from.
|
|
|
|
Members are listed in the order blocks are offloaded: accelerator, then
|
|
host memory, then secondary tiers. Per-source token counts are plain
|
|
``dict[CacheHitSource, int]`` mappings holding only non-zero entries.
|
|
"""
|
|
|
|
DEVICE = "device"
|
|
HOST = "host"
|
|
P2P = "p2p"
|
|
DISK = "disk"
|
|
# Fallback: uninstrumented connector or remote store. Last, so mixing
|
|
# with a known tier never reports the known one.
|
|
EXTERNAL_UNSPECIFIED = "external_unspecified"
|
|
|
|
@classmethod
|
|
def outermost(cls, sources: Iterable[CacheHitSource]) -> CacheHitSource:
|
|
"""The tier in ``sources`` farthest from the accelerator."""
|
|
return max(sources, key=_SOURCE_ORDER.__getitem__)
|
|
|
|
|
|
_SOURCE_ORDER = {source: rank for rank, source in enumerate(CacheHitSource)}
|