Signed-off-by: Amir Friedman <Amir.Friedman1@ibm.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
37 lines
1.2 KiB
Python
37 lines
1.2 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
from dataclasses import dataclass
|
|
|
|
import numpy as np
|
|
|
|
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec, ConfigInfo
|
|
|
|
|
|
class CPUOffloadingMetrics:
|
|
STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
|
|
CPU_CACHE_USAGE_PERC = "vllm:kv_offload_cpu_cache_usage_perc"
|
|
CPU_ALLOCATION_SIZE = "vllm:kv_offload_cpu_allocation_size"
|
|
CPU_CACHE_WRITE_USAGE_PERC = "vllm:kv_offload_cpu_cache_write_usage_perc"
|
|
CPU_CACHE_READ_USAGE_PERC = "vllm:kv_offload_cpu_cache_read_usage_perc"
|
|
|
|
|
|
class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
|
|
"""Spec for loading/storing KV chunks to/from CPU memory.
|
|
|
|
The inherited block_ids field holds chunk indices into the
|
|
CPU cache (not GPU block IDs). The chunk_ids alias exposes
|
|
the same array under the name used by the tiering layer.
|
|
"""
|
|
|
|
@property
|
|
def chunk_ids(self) -> np.ndarray:
|
|
return self.block_ids
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CPUOffloadingInfo(ConfigInfo):
|
|
"""Static facts about the CPU offload tier."""
|
|
|
|
# Chunk slots in the tier. Chunks, not GPU blocks.
|
|
cpu_num_chunks: int
|