# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ExampleSecondaryTierManager: A simple in-memory secondary tier. This implementation provides a minimal secondary tier that stores chunks in memory (using a dictionary) with immediate completion. It serves as a reference for writing new tiers and is useful for testing the TieringOffloadingManager without requiring actual storage or network backends. """ import logging from collections.abc import Iterable from dataclasses import dataclass from typing import TYPE_CHECKING, Any, ClassVar from typing_extensions import override from vllm.v1.kv_offload.base import ( ConfigInfo, ConfigInfoMapping, LookupResult, Medium, OffloadKey, ReqContext, RequestOffloadingContext, ) from vllm.v1.kv_offload.tiering.base import ( JobResult, SecondaryTierManager, TransferJob, ) logger = logging.getLogger(__name__) if TYPE_CHECKING: from vllm.v1.kv_offload.base import OffloadingSpec from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector @dataclass(frozen=True) class ExampleTierInfo(ConfigInfo): """Config facts of one ExampleSecondaryTierManager. A field name is the label name. A secondary tier needs no name of its own for the tier, because the tier label tells the series of the tiers apart. """ example_info: int class ExampleSecondaryTierManager(SecondaryTierManager): """A simple in-memory secondary tier. This implementation: - Stores chunks in a dictionary (key -> True) - Completes transfers immediately (synchronous) """ medium: ClassVar[Medium] = Medium.CPU def __init__( self, offloading_spec: "OffloadingSpec", primary_kv_view: memoryview, tier_type: str, custom_param: int = 0, backpressure_detector: "BackpressureDetector | None" = None, ): """Initialize the example secondary tier. Args: offloading_spec: The offloading spec this tier belongs to. primary_kv_view: Memoryview over the primary tier's KV buffer. tier_type: Name identifying this tier type. custom_param: Dummy parameter demonstrating custom args. backpressure_detector: Optional backpressure detector. """ super().__init__( offloading_spec=offloading_spec, primary_kv_view=primary_kv_view, tier_type=tier_type, backpressure_detector=backpressure_detector, ) self._config_info = ExampleTierInfo(example_info=custom_param) logger.info( "ExampleSecondaryTierManager initialized with custom_param=%d", custom_param ) # key -> True (only care about presence) self.chunks: dict[OffloadKey, bool] = {} # Completed jobs waiting to be retrieved by get_finished_jobs() self.completed_jobs: list[JobResult] = [] assert primary_kv_view.strides is not None self._chunk_size = primary_kv_view.strides[0] @classmethod @override def config_info_keys(cls, tier_config: dict[str, Any]) -> tuple[str, ...]: return ExampleTierInfo.config_info_keys() @override def config_info(self) -> ConfigInfoMapping: return self._config_info.as_config_info() @override def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: """Check whether a chunk exists in this secondary tier. Args: key: Offload key to look up. req_context: Per-request context. Returns: HIT if the chunk is present, MISS if not found. """ return LookupResult.HIT if key in self.chunks else LookupResult.MISS @override def submit_store(self, job_metadata: TransferJob) -> None: """Submit a job to store chunks from primary tier to this tier. Args: job_metadata: Job metadata including job_id, keys, and spec for reading chunks from the primary tier. """ keys = job_metadata.keys chunk_ids = job_metadata.chunk_ids assert len(keys) == len(chunk_ids), ( f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids" ) for key in keys: self.chunks[key] = True self.completed_jobs.append( JobResult( job_id=job_metadata.job_id, success=True, transfer_time=0.0, ) ) @override def submit_load(self, job_metadata: TransferJob) -> None: """Submit a job to load chunks from this tier to primary tier. Args: job_metadata: Job metadata including job_id, keys, and spec for writing chunks into the primary tier. """ keys = job_metadata.keys chunk_ids = job_metadata.chunk_ids assert len(keys) == len(chunk_ids), ( f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids" ) for key in keys: if key not in self.chunks: self.completed_jobs.append( JobResult(job_id=job_metadata.job_id, success=False) ) return self.completed_jobs.append( JobResult( job_id=job_metadata.job_id, success=True, transfer_time=0.0, ) ) @override def get_finished_jobs(self) -> Iterable[JobResult]: """Poll for finished jobs. Returns: Iterable of JobResult objects for all jobs that have finished since the last call. """ result = self.completed_jobs self.completed_jobs = [] return result @override def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: return RequestOffloadingContext() @override def drain_jobs(self) -> None: """Synchronous tier — submit_*() returns only after the operation completes, so there is nothing to wait for.""" return def get_num_chunks(self) -> int: """Get the number of chunks currently stored in this tier.""" return len(self.chunks)