Signed-off-by: Amir Friedman <Amir.Friedman1@ibm.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
206 lines
6.2 KiB
Python
206 lines
6.2 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""ExampleSecondaryTierManager: A simple in-memory secondary tier.
|
|
|
|
This implementation provides a minimal secondary tier that stores chunks
|
|
in memory (using a dictionary) with immediate completion. It serves as a
|
|
reference for writing new tiers and is useful for testing the
|
|
TieringOffloadingManager without requiring actual storage or network backends.
|
|
"""
|
|
|
|
import logging
|
|
from collections.abc import Iterable
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING, Any, ClassVar
|
|
|
|
from typing_extensions import override
|
|
|
|
from vllm.v1.kv_offload.base import (
|
|
ConfigInfo,
|
|
ConfigInfoMapping,
|
|
LookupResult,
|
|
Medium,
|
|
OffloadKey,
|
|
ReqContext,
|
|
RequestOffloadingContext,
|
|
)
|
|
from vllm.v1.kv_offload.tiering.base import (
|
|
JobResult,
|
|
SecondaryTierManager,
|
|
TransferJob,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.v1.kv_offload.base import OffloadingSpec
|
|
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ExampleTierInfo(ConfigInfo):
|
|
"""Config facts of one ExampleSecondaryTierManager.
|
|
|
|
A field name is the label name. A secondary tier needs no name of its own
|
|
for the tier, because the tier label tells the series of the tiers apart.
|
|
"""
|
|
|
|
example_info: int
|
|
|
|
|
|
class ExampleSecondaryTierManager(SecondaryTierManager):
|
|
"""A simple in-memory secondary tier.
|
|
|
|
This implementation:
|
|
- Stores chunks in a dictionary (key -> True)
|
|
- Completes transfers immediately (synchronous)
|
|
"""
|
|
|
|
medium: ClassVar[Medium] = Medium.CPU
|
|
|
|
def __init__(
|
|
self,
|
|
offloading_spec: "OffloadingSpec",
|
|
primary_kv_view: memoryview,
|
|
tier_type: str,
|
|
custom_param: int = 0,
|
|
backpressure_detector: "BackpressureDetector | None" = None,
|
|
):
|
|
"""Initialize the example secondary tier.
|
|
|
|
Args:
|
|
offloading_spec: The offloading spec this tier belongs to.
|
|
primary_kv_view: Memoryview over the primary tier's KV buffer.
|
|
tier_type: Name identifying this tier type.
|
|
custom_param: Dummy parameter demonstrating custom args.
|
|
backpressure_detector: Optional backpressure detector.
|
|
|
|
"""
|
|
super().__init__(
|
|
offloading_spec=offloading_spec,
|
|
primary_kv_view=primary_kv_view,
|
|
tier_type=tier_type,
|
|
backpressure_detector=backpressure_detector,
|
|
)
|
|
|
|
self._config_info = ExampleTierInfo(example_info=custom_param)
|
|
|
|
logger.info(
|
|
"ExampleSecondaryTierManager initialized with custom_param=%d", custom_param
|
|
)
|
|
|
|
# key -> True (only care about presence)
|
|
self.chunks: dict[OffloadKey, bool] = {}
|
|
|
|
# Completed jobs waiting to be retrieved by get_finished_jobs()
|
|
self.completed_jobs: list[JobResult] = []
|
|
assert primary_kv_view.strides is not None
|
|
self._chunk_size = primary_kv_view.strides[0]
|
|
|
|
@classmethod
|
|
@override
|
|
def config_info_keys(cls, tier_config: dict[str, Any]) -> tuple[str, ...]:
|
|
return ExampleTierInfo.config_info_keys()
|
|
|
|
@override
|
|
def config_info(self) -> ConfigInfoMapping:
|
|
return self._config_info.as_config_info()
|
|
|
|
@override
|
|
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
|
"""Check whether a chunk exists in this secondary tier.
|
|
|
|
Args:
|
|
key: Offload key to look up.
|
|
req_context: Per-request context.
|
|
|
|
Returns:
|
|
HIT if the chunk is present, MISS if not found.
|
|
|
|
"""
|
|
return LookupResult.HIT if key in self.chunks else LookupResult.MISS
|
|
|
|
@override
|
|
def submit_store(self, job_metadata: TransferJob) -> None:
|
|
"""Submit a job to store chunks from primary tier to this tier.
|
|
|
|
Args:
|
|
job_metadata: Job metadata including job_id, keys, and
|
|
spec for reading chunks from the primary tier.
|
|
|
|
"""
|
|
keys = job_metadata.keys
|
|
chunk_ids = job_metadata.chunk_ids
|
|
|
|
assert len(keys) == len(chunk_ids), (
|
|
f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids"
|
|
)
|
|
|
|
for key in keys:
|
|
self.chunks[key] = True
|
|
self.completed_jobs.append(
|
|
JobResult(
|
|
job_id=job_metadata.job_id,
|
|
success=True,
|
|
transfer_time=0.0,
|
|
)
|
|
)
|
|
|
|
@override
|
|
def submit_load(self, job_metadata: TransferJob) -> None:
|
|
"""Submit a job to load chunks from this tier to primary tier.
|
|
|
|
Args:
|
|
job_metadata: Job metadata including job_id, keys, and
|
|
spec for writing chunks into the primary tier.
|
|
|
|
"""
|
|
keys = job_metadata.keys
|
|
chunk_ids = job_metadata.chunk_ids
|
|
|
|
assert len(keys) == len(chunk_ids), (
|
|
f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids"
|
|
)
|
|
|
|
for key in keys:
|
|
if key not in self.chunks:
|
|
self.completed_jobs.append(
|
|
JobResult(job_id=job_metadata.job_id, success=False)
|
|
)
|
|
return
|
|
|
|
self.completed_jobs.append(
|
|
JobResult(
|
|
job_id=job_metadata.job_id,
|
|
success=True,
|
|
transfer_time=0.0,
|
|
)
|
|
)
|
|
|
|
@override
|
|
def get_finished_jobs(self) -> Iterable[JobResult]:
|
|
"""Poll for finished jobs.
|
|
|
|
Returns:
|
|
Iterable of JobResult objects for all jobs that have
|
|
finished since the last call.
|
|
|
|
"""
|
|
result = self.completed_jobs
|
|
self.completed_jobs = []
|
|
return result
|
|
|
|
@override
|
|
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
|
return RequestOffloadingContext()
|
|
|
|
@override
|
|
def drain_jobs(self) -> None:
|
|
"""Synchronous tier — submit_*() returns only after the operation
|
|
completes, so there is nothing to wait for."""
|
|
return
|
|
|
|
def get_num_chunks(self) -> int:
|
|
"""Get the number of chunks currently stored in this tier."""
|
|
return len(self.chunks)
|