[kv_offload] Add @override decorators to subclass method implementations (#44177)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, NewType
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
@@ -311,6 +312,7 @@ class GPULoadStoreSpec(BlockIDsLoadStoreSpec):
|
||||
self.block_indices: Sequence[int] = block_indices
|
||||
|
||||
@staticmethod
|
||||
@override
|
||||
def medium() -> str:
|
||||
return "GPU"
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
|
||||
|
||||
|
||||
@@ -9,5 +11,6 @@ class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@override
|
||||
def medium() -> str:
|
||||
return "CPU"
|
||||
|
||||
@@ -6,6 +6,7 @@ from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
@@ -179,6 +180,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
# list of CUDA events available for re-use
|
||||
self._event_pool: list[torch.Event] = []
|
||||
|
||||
@override
|
||||
def transfer_async(self, job_id: int, transfer_spec: TransferSpec) -> bool:
|
||||
src_spec, dst_spec = transfer_spec
|
||||
assert isinstance(src_spec, BlockIDsLoadStoreSpec)
|
||||
@@ -348,6 +350,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
# success
|
||||
return True
|
||||
|
||||
@override
|
||||
def get_finished(self) -> list[TransferResult]:
|
||||
results: list[TransferResult] = []
|
||||
while self._transfers and self._transfers[0].end_event.query():
|
||||
@@ -370,12 +373,14 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
|
||||
del self._transfer_events[transfer.job_id]
|
||||
return results
|
||||
|
||||
@override
|
||||
def wait(self, job_ids: set[int]):
|
||||
for job_id in job_ids:
|
||||
event = self._transfer_events.get(job_id)
|
||||
if event is not None:
|
||||
event.synchronize()
|
||||
|
||||
@override
|
||||
def shutdown(self) -> None:
|
||||
while self._transfers:
|
||||
transfer = self._transfers.popleft()
|
||||
|
||||
@@ -4,6 +4,8 @@ from collections import OrderedDict
|
||||
from collections.abc import Collection, Iterable
|
||||
from typing import Literal
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LoadStoreSpec,
|
||||
OffloadingEvent,
|
||||
@@ -95,9 +97,11 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
|
||||
# --- OffloadingManager interface ---
|
||||
|
||||
@override
|
||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||
return RequestOffloadingContext()
|
||||
|
||||
@override
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
if self.counts is not None:
|
||||
if key in self.counts:
|
||||
@@ -114,6 +118,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
return None # write in-flight; caller should retry
|
||||
return True
|
||||
|
||||
@override
|
||||
def prepare_load(
|
||||
self,
|
||||
keys: Collection[OffloadKey],
|
||||
@@ -128,9 +133,11 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
blocks.append(block)
|
||||
return self._get_load_store_spec(keys, blocks)
|
||||
|
||||
@override
|
||||
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext) -> None:
|
||||
self._policy.touch(keys)
|
||||
|
||||
@override
|
||||
def complete_load(
|
||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||
) -> None:
|
||||
@@ -140,6 +147,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
assert block.ref_cnt > 0, f"Block {key!r} ref_cnt is already 0"
|
||||
block.ref_cnt -= 1
|
||||
|
||||
@override
|
||||
def prepare_store(
|
||||
self,
|
||||
keys: Collection[OffloadKey],
|
||||
@@ -197,6 +205,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
evicted_keys=to_evict,
|
||||
)
|
||||
|
||||
@override
|
||||
def complete_store(
|
||||
self,
|
||||
keys: Collection[OffloadKey],
|
||||
@@ -227,6 +236,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
)
|
||||
)
|
||||
|
||||
@override
|
||||
def reset_cache(self) -> None:
|
||||
# Clear ALL blocks unconditionally. The scheduler's _stale_job_threshold
|
||||
# guarantees that complete_load / complete_store are never called for
|
||||
@@ -238,6 +248,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
self._free_list.clear()
|
||||
self._num_allocated_blocks = 0
|
||||
|
||||
@override
|
||||
def take_events(self) -> Iterable[OffloadingEvent]:
|
||||
if self.events is not None:
|
||||
yield from self.events
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Iterable
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.v1.kv_offload.base import OffloadKey
|
||||
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
|
||||
|
||||
@@ -54,18 +56,22 @@ class ARCCachePolicy(CachePolicy):
|
||||
self.b1: OrderedDict[OffloadKey, None] = OrderedDict()
|
||||
self.b2: OrderedDict[OffloadKey, None] = OrderedDict()
|
||||
|
||||
@override
|
||||
def get(self, key: OffloadKey) -> BlockStatus | None:
|
||||
return self.t1.get(key) or self.t2.get(key)
|
||||
|
||||
@override
|
||||
def insert(self, key: OffloadKey, block: BlockStatus) -> None:
|
||||
self.t1[key] = block
|
||||
self.b1.pop(key, None)
|
||||
self.b2.pop(key, None)
|
||||
|
||||
@override
|
||||
def remove(self, key: OffloadKey) -> None:
|
||||
if self.t1.pop(key, None) is None:
|
||||
self.t2.pop(key, None)
|
||||
|
||||
@override
|
||||
def touch(self, keys: Iterable[OffloadKey]) -> None:
|
||||
for key in reversed(list(keys)):
|
||||
if key in self.t1:
|
||||
@@ -94,6 +100,7 @@ class ARCCachePolicy(CachePolicy):
|
||||
# move to MRU position (end) to keep it fresh in the ghost list
|
||||
self.b2.move_to_end(key)
|
||||
|
||||
@override
|
||||
def clear(self) -> None:
|
||||
self.t1.clear()
|
||||
self.t2.clear()
|
||||
@@ -101,6 +108,7 @@ class ARCCachePolicy(CachePolicy):
|
||||
self.b2.clear()
|
||||
self.target_t1_size = 0.0
|
||||
|
||||
@override
|
||||
def evict(
|
||||
self, n: int, protected: set[OffloadKey]
|
||||
) -> list[tuple[OffloadKey, BlockStatus]] | None:
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Iterable
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.v1.kv_offload.base import OffloadKey
|
||||
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
|
||||
|
||||
@@ -14,23 +16,29 @@ class LRUCachePolicy(CachePolicy):
|
||||
# cache_capacity unused by LRU but accepted for a uniform constructor
|
||||
self.blocks: OrderedDict[OffloadKey, BlockStatus] = OrderedDict()
|
||||
|
||||
@override
|
||||
def get(self, key: OffloadKey) -> BlockStatus | None:
|
||||
return self.blocks.get(key)
|
||||
|
||||
@override
|
||||
def insert(self, key: OffloadKey, block: BlockStatus) -> None:
|
||||
self.blocks[key] = block
|
||||
|
||||
@override
|
||||
def remove(self, key: OffloadKey) -> None:
|
||||
del self.blocks[key]
|
||||
|
||||
@override
|
||||
def touch(self, keys: Iterable[OffloadKey]) -> None:
|
||||
for key in reversed(list(keys)):
|
||||
if key in self.blocks:
|
||||
self.blocks.move_to_end(key)
|
||||
|
||||
@override
|
||||
def clear(self) -> None:
|
||||
self.blocks.clear()
|
||||
|
||||
@override
|
||||
def evict(
|
||||
self, n: int, protected: set[OffloadKey]
|
||||
) -> list[tuple[OffloadKey, BlockStatus]] | None:
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterator
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
@@ -57,6 +59,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
|
||||
self.eviction_policy: str = self.extra_config.get("eviction_policy", "lru")
|
||||
|
||||
@override
|
||||
def get_manager(self) -> OffloadingManager:
|
||||
if not self._manager:
|
||||
kv_events_config = self.vllm_config.kv_events_config
|
||||
@@ -88,6 +91,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
num_cpu_blocks=self.num_blocks,
|
||||
)
|
||||
|
||||
@override
|
||||
def get_handlers(
|
||||
self, kv_caches: CanonicalKVCaches
|
||||
) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]:
|
||||
|
||||
@@ -13,6 +13,8 @@ import logging
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, RequestOffloadingContext
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobMetadata,
|
||||
@@ -64,6 +66,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
# Completed jobs waiting to be retrieved by get_finished_jobs()
|
||||
self.completed_jobs: list[JobResult] = []
|
||||
|
||||
@override
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
"""
|
||||
Check whether a block exists in this secondary tier.
|
||||
@@ -77,6 +80,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
"""
|
||||
return key in self.blocks
|
||||
|
||||
@override
|
||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
||||
"""
|
||||
Submit a job to store blocks from primary tier to this tier.
|
||||
@@ -96,6 +100,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
self.blocks[key] = True
|
||||
self.completed_jobs.append(JobResult(job_id=job_metadata.job_id, success=True))
|
||||
|
||||
@override
|
||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
||||
"""
|
||||
Submit a job to load blocks from this tier to primary tier.
|
||||
@@ -120,6 +125,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
|
||||
self.completed_jobs.append(JobResult(job_id=job_metadata.job_id, success=True))
|
||||
|
||||
@override
|
||||
def get_finished_jobs(self) -> Iterable[JobResult]:
|
||||
"""
|
||||
Poll for finished jobs.
|
||||
@@ -132,6 +138,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
self.completed_jobs = []
|
||||
return result
|
||||
|
||||
@override
|
||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||
return RequestOffloadingContext()
|
||||
|
||||
|
||||
@@ -21,6 +21,8 @@ import os
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
@@ -101,14 +103,17 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
thread_name_prefix="vllm_kv_py_fs",
|
||||
)
|
||||
|
||||
@override
|
||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||
return RequestOffloadingContext()
|
||||
|
||||
@override
|
||||
def lookup(
|
||||
self, key: OffloadKey, req_context: ReqContext | None = None
|
||||
) -> bool | None:
|
||||
return os.path.exists(self.file_mapper.get_file_name(key))
|
||||
|
||||
@override
|
||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
||||
tasks = (
|
||||
functools.partial(
|
||||
@@ -122,6 +127,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
)
|
||||
self._pool.enqueue_store(job_metadata.job_id, len(job_metadata.keys), tasks)
|
||||
|
||||
@override
|
||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
||||
tasks = (
|
||||
functools.partial(
|
||||
@@ -135,6 +141,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
)
|
||||
self._pool.enqueue_load(job_metadata.job_id, len(job_metadata.keys), tasks)
|
||||
|
||||
@override
|
||||
def get_finished_jobs(self) -> Iterable[JobResult]:
|
||||
"""
|
||||
Collect completed jobs from the finished-jobs queue.
|
||||
@@ -144,6 +151,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
for job_id, success in self._pool.get_finished()
|
||||
)
|
||||
|
||||
@override
|
||||
def shutdown(self) -> None:
|
||||
"""
|
||||
Release resources held by this tier.
|
||||
|
||||
@@ -25,6 +25,7 @@ from collections.abc import Collection, Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import numpy as np
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import (
|
||||
@@ -100,6 +101,7 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
||||
"""
|
||||
return self._kv_memoryview
|
||||
|
||||
@override
|
||||
def shutdown(self) -> None:
|
||||
super().shutdown()
|
||||
self._kv_memoryview.release()
|
||||
@@ -222,6 +224,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
job_metadata.keys, job_metadata.req_context
|
||||
)
|
||||
|
||||
@override
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
"""
|
||||
Check whether a single block is offloaded and ready.
|
||||
@@ -338,6 +341,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
|
||||
self._pending_load_submissions.clear()
|
||||
|
||||
@override
|
||||
def prepare_load(
|
||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||
) -> LoadStoreSpec:
|
||||
@@ -362,6 +366,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
|
||||
return self.primary_tier.prepare_load(keys, req_context)
|
||||
|
||||
@override
|
||||
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
||||
"""
|
||||
Mark blocks as recently used in all tiers.
|
||||
@@ -374,6 +379,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
for tier in self.secondary_tiers:
|
||||
tier.touch(keys, req_context)
|
||||
|
||||
@override
|
||||
def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
||||
"""
|
||||
Mark blocks as done loading from primary tier to GPU.
|
||||
@@ -387,6 +393,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
"""
|
||||
self.primary_tier.complete_load(keys, req_context)
|
||||
|
||||
@override
|
||||
def prepare_store(
|
||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||
) -> PrepareStoreOutput | None:
|
||||
@@ -470,6 +477,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
self._transfer_jobs[job_id] = job_metadata
|
||||
tier.submit_store(job_metadata)
|
||||
|
||||
@override
|
||||
def complete_store(
|
||||
self,
|
||||
keys: Collection[OffloadKey],
|
||||
@@ -528,6 +536,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
# Note: The async transfers are now in flight. Their completion is
|
||||
# tracked via get_finished_jobs() / _maybe_process_finished_jobs().
|
||||
|
||||
@override
|
||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||
"""
|
||||
Query each secondary tier for its offload policy preference.
|
||||
@@ -547,12 +556,14 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
)
|
||||
return RequestOffloadingContext(policy=policy)
|
||||
|
||||
@override
|
||||
def on_request_finished(self, req_context: ReqContext) -> None:
|
||||
self.primary_tier.on_request_finished(req_context)
|
||||
for tier in self.secondary_tiers:
|
||||
tier.on_request_finished(req_context)
|
||||
self._request_level_tiers.pop(req_context.req_id, None)
|
||||
|
||||
@override
|
||||
def take_events(self) -> Iterable[OffloadingEvent]:
|
||||
"""
|
||||
End-of-step hook: flush deferred work, yield events, reset per-step state.
|
||||
@@ -585,6 +596,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
|
||||
yield from self.primary_tier.take_events()
|
||||
|
||||
@override
|
||||
def shutdown(self) -> None:
|
||||
"""Shutdown all tiers and release resources."""
|
||||
for tier in self.secondary_tiers:
|
||||
|
||||
Reference in New Issue
Block a user