cdaa40d2a8
Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com> Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
18 lines
653 B
Python
18 lines
653 B
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
|
|
|
|
|
|
class CPUOffloadingMetrics:
|
|
STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
|
|
CPU_CACHE_USAGE_PERC = "vllm:kv_offload_cpu_cache_usage_perc"
|
|
CPU_ALLOCATION_SIZE = "vllm:kv_offload_cpu_allocation_size"
|
|
CPU_CACHE_WRITE_USAGE_PERC = "vllm:kv_offload_cpu_cache_write_usage_perc"
|
|
CPU_CACHE_READ_USAGE_PERC = "vllm:kv_offload_cpu_cache_read_usage_perc"
|
|
|
|
|
|
class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
|
|
"""
|
|
Spec for loading/storing a KV block to CPU memory.
|
|
"""
|