Files

206 lines
6.2 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ExampleSecondaryTierManager: A simple in-memory secondary tier.
This implementation provides a minimal secondary tier that stores chunks
in memory (using a dictionary) with immediate completion. It serves as a
reference for writing new tiers and is useful for testing the
TieringOffloadingManager without requiring actual storage or network backends.
"""
import logging
from collections.abc import Iterable
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, ClassVar
from typing_extensions import override
from vllm.v1.kv_offload.base import (
ConfigInfo,
ConfigInfoMapping,
LookupResult,
Medium,
OffloadKey,
ReqContext,
RequestOffloadingContext,
)
from vllm.v1.kv_offload.tiering.base import (
JobResult,
SecondaryTierManager,
TransferJob,
)
logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from vllm.v1.kv_offload.base import OffloadingSpec
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
@dataclass(frozen=True)
class ExampleTierInfo(ConfigInfo):
"""Config facts of one ExampleSecondaryTierManager.
A field name is the label name. A secondary tier needs no name of its own
for the tier, because the tier label tells the series of the tiers apart.
"""
example_info: int
class ExampleSecondaryTierManager(SecondaryTierManager):
"""A simple in-memory secondary tier.
This implementation:
- Stores chunks in a dictionary (key -> True)
- Completes transfers immediately (synchronous)
"""
medium: ClassVar[Medium] = Medium.CPU
def __init__(
self,
offloading_spec: "OffloadingSpec",
primary_kv_view: memoryview,
tier_type: str,
custom_param: int = 0,
backpressure_detector: "BackpressureDetector | None" = None,
):
"""Initialize the example secondary tier.
Args:
offloading_spec: The offloading spec this tier belongs to.
primary_kv_view: Memoryview over the primary tier's KV buffer.
tier_type: Name identifying this tier type.
custom_param: Dummy parameter demonstrating custom args.
backpressure_detector: Optional backpressure detector.
"""
super().__init__(
offloading_spec=offloading_spec,
primary_kv_view=primary_kv_view,
tier_type=tier_type,
backpressure_detector=backpressure_detector,
)
self._config_info = ExampleTierInfo(example_info=custom_param)
logger.info(
"ExampleSecondaryTierManager initialized with custom_param=%d", custom_param
)
# key -> True (only care about presence)
self.chunks: dict[OffloadKey, bool] = {}
# Completed jobs waiting to be retrieved by get_finished_jobs()
self.completed_jobs: list[JobResult] = []
assert primary_kv_view.strides is not None
self._chunk_size = primary_kv_view.strides[0]
@classmethod
@override
def config_info_keys(cls, tier_config: dict[str, Any]) -> tuple[str, ...]:
return ExampleTierInfo.config_info_keys()
@override
def config_info(self) -> ConfigInfoMapping:
return self._config_info.as_config_info()
@override
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
"""Check whether a chunk exists in this secondary tier.
Args:
key: Offload key to look up.
req_context: Per-request context.
Returns:
HIT if the chunk is present, MISS if not found.
"""
return LookupResult.HIT if key in self.chunks else LookupResult.MISS
@override
def submit_store(self, job_metadata: TransferJob) -> None:
"""Submit a job to store chunks from primary tier to this tier.
Args:
job_metadata: Job metadata including job_id, keys, and
spec for reading chunks from the primary tier.
"""
keys = job_metadata.keys
chunk_ids = job_metadata.chunk_ids
assert len(keys) == len(chunk_ids), (
f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids"
)
for key in keys:
self.chunks[key] = True
self.completed_jobs.append(
JobResult(
job_id=job_metadata.job_id,
success=True,
transfer_time=0.0,
)
)
@override
def submit_load(self, job_metadata: TransferJob) -> None:
"""Submit a job to load chunks from this tier to primary tier.
Args:
job_metadata: Job metadata including job_id, keys, and
spec for writing chunks into the primary tier.
"""
keys = job_metadata.keys
chunk_ids = job_metadata.chunk_ids
assert len(keys) == len(chunk_ids), (
f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids"
)
for key in keys:
if key not in self.chunks:
self.completed_jobs.append(
JobResult(job_id=job_metadata.job_id, success=False)
)
return
self.completed_jobs.append(
JobResult(
job_id=job_metadata.job_id,
success=True,
transfer_time=0.0,
)
)
@override
def get_finished_jobs(self) -> Iterable[JobResult]:
"""Poll for finished jobs.
Returns:
Iterable of JobResult objects for all jobs that have
finished since the last call.
"""
result = self.completed_jobs
self.completed_jobs = []
return result
@override
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
return RequestOffloadingContext()
@override
def drain_jobs(self) -> None:
"""Synchronous tier — submit_*() returns only after the operation
completes, so there is nothing to wait for."""
return
def get_num_chunks(self) -> int:
"""Get the number of chunks currently stored in this tier."""
return len(self.chunks)