27 lines
944 B
Python
27 lines
944 B
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
import numpy as np
|
|
|
|
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
|
|
|
|
|
|
class CPUOffloadingMetrics:
|
|
STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
|
|
CPU_CACHE_USAGE_PERC = "vllm:kv_offload_cpu_cache_usage_perc"
|
|
CPU_ALLOCATION_SIZE = "vllm:kv_offload_cpu_allocation_size"
|
|
CPU_CACHE_WRITE_USAGE_PERC = "vllm:kv_offload_cpu_cache_write_usage_perc"
|
|
CPU_CACHE_READ_USAGE_PERC = "vllm:kv_offload_cpu_cache_read_usage_perc"
|
|
|
|
|
|
class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
|
|
"""Spec for loading/storing KV chunks to/from CPU memory.
|
|
|
|
The inherited block_ids field holds chunk indices into the
|
|
CPU cache (not GPU block IDs). The chunk_ids alias exposes
|
|
the same array under the name used by the tiering layer.
|
|
"""
|
|
|
|
@property
|
|
def chunk_ids(self) -> np.ndarray:
|
|
return self.block_ids
|