From 93da882e73787fb7c4d5e122ea5c48ff3ad1e0c3 Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Tue, 2 Jun 2026 11:07:47 +0300 Subject: [PATCH] [kv_offload] Add `@override` decorators to subclass method implementations (#44177) Signed-off-by: Ronen Schaffer --- vllm/v1/kv_offload/base.py | 2 ++ vllm/v1/kv_offload/cpu/common.py | 3 +++ vllm/v1/kv_offload/cpu/gpu_worker.py | 5 +++++ vllm/v1/kv_offload/cpu/manager.py | 11 +++++++++++ vllm/v1/kv_offload/cpu/policies/arc.py | 8 ++++++++ vllm/v1/kv_offload/cpu/policies/lru.py | 8 ++++++++ vllm/v1/kv_offload/cpu/spec.py | 4 ++++ vllm/v1/kv_offload/tiering/example/manager.py | 7 +++++++ vllm/v1/kv_offload/tiering/fs/manager.py | 8 ++++++++ vllm/v1/kv_offload/tiering/manager.py | 12 ++++++++++++ 10 files changed, 68 insertions(+) diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index ec168de891..ecde261ff2 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, NewType import numpy as np import torch +from typing_extensions import override from vllm.logger import init_logger from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes @@ -311,6 +312,7 @@ class GPULoadStoreSpec(BlockIDsLoadStoreSpec): self.block_indices: Sequence[int] = block_indices @staticmethod + @override def medium() -> str: return "GPU" diff --git a/vllm/v1/kv_offload/cpu/common.py b/vllm/v1/kv_offload/cpu/common.py index cf5b2b39dd..42f576bb70 100644 --- a/vllm/v1/kv_offload/cpu/common.py +++ b/vllm/v1/kv_offload/cpu/common.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing_extensions import override + from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec @@ -9,5 +11,6 @@ class CPULoadStoreSpec(BlockIDsLoadStoreSpec): """ @staticmethod + @override def medium() -> str: return "CPU" diff --git a/vllm/v1/kv_offload/cpu/gpu_worker.py b/vllm/v1/kv_offload/cpu/gpu_worker.py index 119778368c..a57845bd44 100644 --- a/vllm/v1/kv_offload/cpu/gpu_worker.py +++ b/vllm/v1/kv_offload/cpu/gpu_worker.py @@ -6,6 +6,7 @@ from dataclasses import dataclass import numpy as np import torch +from typing_extensions import override from vllm import _custom_ops as ops from vllm.logger import init_logger @@ -179,6 +180,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): # list of CUDA events available for re-use self._event_pool: list[torch.Event] = [] + @override def transfer_async(self, job_id: int, transfer_spec: TransferSpec) -> bool: src_spec, dst_spec = transfer_spec assert isinstance(src_spec, BlockIDsLoadStoreSpec) @@ -348,6 +350,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): # success return True + @override def get_finished(self) -> list[TransferResult]: results: list[TransferResult] = [] while self._transfers and self._transfers[0].end_event.query(): @@ -370,12 +373,14 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): del self._transfer_events[transfer.job_id] return results + @override def wait(self, job_ids: set[int]): for job_id in job_ids: event = self._transfer_events.get(job_id) if event is not None: event.synchronize() + @override def shutdown(self) -> None: while self._transfers: transfer = self._transfers.popleft() diff --git a/vllm/v1/kv_offload/cpu/manager.py b/vllm/v1/kv_offload/cpu/manager.py index 4700e250c5..a1d3a30ebb 100644 --- a/vllm/v1/kv_offload/cpu/manager.py +++ b/vllm/v1/kv_offload/cpu/manager.py @@ -4,6 +4,8 @@ from collections import OrderedDict from collections.abc import Collection, Iterable from typing import Literal +from typing_extensions import override + from vllm.v1.kv_offload.base import ( LoadStoreSpec, OffloadingEvent, @@ -95,9 +97,11 @@ class CPUOffloadingManager(OffloadingManager): # --- OffloadingManager interface --- + @override def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: return RequestOffloadingContext() + @override def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None: if self.counts is not None: if key in self.counts: @@ -114,6 +118,7 @@ class CPUOffloadingManager(OffloadingManager): return None # write in-flight; caller should retry return True + @override def prepare_load( self, keys: Collection[OffloadKey], @@ -128,9 +133,11 @@ class CPUOffloadingManager(OffloadingManager): blocks.append(block) return self._get_load_store_spec(keys, blocks) + @override def touch(self, keys: Collection[OffloadKey], req_context: ReqContext) -> None: self._policy.touch(keys) + @override def complete_load( self, keys: Collection[OffloadKey], req_context: ReqContext ) -> None: @@ -140,6 +147,7 @@ class CPUOffloadingManager(OffloadingManager): assert block.ref_cnt > 0, f"Block {key!r} ref_cnt is already 0" block.ref_cnt -= 1 + @override def prepare_store( self, keys: Collection[OffloadKey], @@ -197,6 +205,7 @@ class CPUOffloadingManager(OffloadingManager): evicted_keys=to_evict, ) + @override def complete_store( self, keys: Collection[OffloadKey], @@ -227,6 +236,7 @@ class CPUOffloadingManager(OffloadingManager): ) ) + @override def reset_cache(self) -> None: # Clear ALL blocks unconditionally. The scheduler's _stale_job_threshold # guarantees that complete_load / complete_store are never called for @@ -238,6 +248,7 @@ class CPUOffloadingManager(OffloadingManager): self._free_list.clear() self._num_allocated_blocks = 0 + @override def take_events(self) -> Iterable[OffloadingEvent]: if self.events is not None: yield from self.events diff --git a/vllm/v1/kv_offload/cpu/policies/arc.py b/vllm/v1/kv_offload/cpu/policies/arc.py index 5b01815c2d..7d22e51865 100644 --- a/vllm/v1/kv_offload/cpu/policies/arc.py +++ b/vllm/v1/kv_offload/cpu/policies/arc.py @@ -3,6 +3,8 @@ from collections import OrderedDict from collections.abc import Iterable +from typing_extensions import override + from vllm.v1.kv_offload.base import OffloadKey from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy @@ -54,18 +56,22 @@ class ARCCachePolicy(CachePolicy): self.b1: OrderedDict[OffloadKey, None] = OrderedDict() self.b2: OrderedDict[OffloadKey, None] = OrderedDict() + @override def get(self, key: OffloadKey) -> BlockStatus | None: return self.t1.get(key) or self.t2.get(key) + @override def insert(self, key: OffloadKey, block: BlockStatus) -> None: self.t1[key] = block self.b1.pop(key, None) self.b2.pop(key, None) + @override def remove(self, key: OffloadKey) -> None: if self.t1.pop(key, None) is None: self.t2.pop(key, None) + @override def touch(self, keys: Iterable[OffloadKey]) -> None: for key in reversed(list(keys)): if key in self.t1: @@ -94,6 +100,7 @@ class ARCCachePolicy(CachePolicy): # move to MRU position (end) to keep it fresh in the ghost list self.b2.move_to_end(key) + @override def clear(self) -> None: self.t1.clear() self.t2.clear() @@ -101,6 +108,7 @@ class ARCCachePolicy(CachePolicy): self.b2.clear() self.target_t1_size = 0.0 + @override def evict( self, n: int, protected: set[OffloadKey] ) -> list[tuple[OffloadKey, BlockStatus]] | None: diff --git a/vllm/v1/kv_offload/cpu/policies/lru.py b/vllm/v1/kv_offload/cpu/policies/lru.py index 51680d8bcc..75fbc6015e 100644 --- a/vllm/v1/kv_offload/cpu/policies/lru.py +++ b/vllm/v1/kv_offload/cpu/policies/lru.py @@ -3,6 +3,8 @@ from collections import OrderedDict from collections.abc import Iterable +from typing_extensions import override + from vllm.v1.kv_offload.base import OffloadKey from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy @@ -14,23 +16,29 @@ class LRUCachePolicy(CachePolicy): # cache_capacity unused by LRU but accepted for a uniform constructor self.blocks: OrderedDict[OffloadKey, BlockStatus] = OrderedDict() + @override def get(self, key: OffloadKey) -> BlockStatus | None: return self.blocks.get(key) + @override def insert(self, key: OffloadKey, block: BlockStatus) -> None: self.blocks[key] = block + @override def remove(self, key: OffloadKey) -> None: del self.blocks[key] + @override def touch(self, keys: Iterable[OffloadKey]) -> None: for key in reversed(list(keys)): if key in self.blocks: self.blocks.move_to_end(key) + @override def clear(self) -> None: self.blocks.clear() + @override def evict( self, n: int, protected: set[OffloadKey] ) -> list[tuple[OffloadKey, BlockStatus]] | None: diff --git a/vllm/v1/kv_offload/cpu/spec.py b/vllm/v1/kv_offload/cpu/spec.py index 6d17d5317f..053f5a140e 100644 --- a/vllm/v1/kv_offload/cpu/spec.py +++ b/vllm/v1/kv_offload/cpu/spec.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Iterator +from typing_extensions import override + from vllm.config import VllmConfig from vllm.platforms import current_platform from vllm.v1.kv_cache_interface import KVCacheConfig @@ -57,6 +59,7 @@ class CPUOffloadingSpec(OffloadingSpec): self.eviction_policy: str = self.extra_config.get("eviction_policy", "lru") + @override def get_manager(self) -> OffloadingManager: if not self._manager: kv_events_config = self.vllm_config.kv_events_config @@ -88,6 +91,7 @@ class CPUOffloadingSpec(OffloadingSpec): num_cpu_blocks=self.num_blocks, ) + @override def get_handlers( self, kv_caches: CanonicalKVCaches ) -> Iterator[tuple[type[LoadStoreSpec], type[LoadStoreSpec], OffloadingHandler]]: diff --git a/vllm/v1/kv_offload/tiering/example/manager.py b/vllm/v1/kv_offload/tiering/example/manager.py index 29ec627ca6..caf1d2c71b 100644 --- a/vllm/v1/kv_offload/tiering/example/manager.py +++ b/vllm/v1/kv_offload/tiering/example/manager.py @@ -13,6 +13,8 @@ import logging from collections.abc import Iterable from typing import TYPE_CHECKING +from typing_extensions import override + from vllm.v1.kv_offload.base import OffloadKey, ReqContext, RequestOffloadingContext from vllm.v1.kv_offload.tiering.base import ( JobMetadata, @@ -64,6 +66,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager): # Completed jobs waiting to be retrieved by get_finished_jobs() self.completed_jobs: list[JobResult] = [] + @override def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None: """ Check whether a block exists in this secondary tier. @@ -77,6 +80,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager): """ return key in self.blocks + @override def submit_store(self, job_metadata: JobMetadata) -> None: """ Submit a job to store blocks from primary tier to this tier. @@ -96,6 +100,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager): self.blocks[key] = True self.completed_jobs.append(JobResult(job_id=job_metadata.job_id, success=True)) + @override def submit_load(self, job_metadata: JobMetadata) -> None: """ Submit a job to load blocks from this tier to primary tier. @@ -120,6 +125,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager): self.completed_jobs.append(JobResult(job_id=job_metadata.job_id, success=True)) + @override def get_finished_jobs(self) -> Iterable[JobResult]: """ Poll for finished jobs. @@ -132,6 +138,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager): self.completed_jobs = [] return result + @override def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: return RequestOffloadingContext() diff --git a/vllm/v1/kv_offload/tiering/fs/manager.py b/vllm/v1/kv_offload/tiering/fs/manager.py index 1d37983d55..c2bc720275 100644 --- a/vllm/v1/kv_offload/tiering/fs/manager.py +++ b/vllm/v1/kv_offload/tiering/fs/manager.py @@ -21,6 +21,8 @@ import os from collections.abc import Iterable from typing import TYPE_CHECKING +from typing_extensions import override + from vllm.logger import init_logger from vllm.v1.kv_offload.base import OffloadKey, ReqContext from vllm.v1.kv_offload.file_mapper import FileMapper @@ -101,14 +103,17 @@ class FileSystemTierManager(SecondaryTierManager): thread_name_prefix="vllm_kv_py_fs", ) + @override def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: return RequestOffloadingContext() + @override def lookup( self, key: OffloadKey, req_context: ReqContext | None = None ) -> bool | None: return os.path.exists(self.file_mapper.get_file_name(key)) + @override def submit_store(self, job_metadata: JobMetadata) -> None: tasks = ( functools.partial( @@ -122,6 +127,7 @@ class FileSystemTierManager(SecondaryTierManager): ) self._pool.enqueue_store(job_metadata.job_id, len(job_metadata.keys), tasks) + @override def submit_load(self, job_metadata: JobMetadata) -> None: tasks = ( functools.partial( @@ -135,6 +141,7 @@ class FileSystemTierManager(SecondaryTierManager): ) self._pool.enqueue_load(job_metadata.job_id, len(job_metadata.keys), tasks) + @override def get_finished_jobs(self) -> Iterable[JobResult]: """ Collect completed jobs from the finished-jobs queue. @@ -144,6 +151,7 @@ class FileSystemTierManager(SecondaryTierManager): for job_id, success in self._pool.get_finished() ) + @override def shutdown(self) -> None: """ Release resources held by this tier. diff --git a/vllm/v1/kv_offload/tiering/manager.py b/vllm/v1/kv_offload/tiering/manager.py index ae8121b4aa..932c764425 100644 --- a/vllm/v1/kv_offload/tiering/manager.py +++ b/vllm/v1/kv_offload/tiering/manager.py @@ -25,6 +25,7 @@ from collections.abc import Collection, Iterable, Sequence from dataclasses import dataclass, field import numpy as np +from typing_extensions import override from vllm.logger import init_logger from vllm.v1.kv_offload.base import ( @@ -100,6 +101,7 @@ class CPUPrimaryTierOffloadingManager(CPUOffloadingManager): """ return self._kv_memoryview + @override def shutdown(self) -> None: super().shutdown() self._kv_memoryview.release() @@ -222,6 +224,7 @@ class TieringOffloadingManager(OffloadingManager): job_metadata.keys, job_metadata.req_context ) + @override def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None: """ Check whether a single block is offloaded and ready. @@ -338,6 +341,7 @@ class TieringOffloadingManager(OffloadingManager): self._pending_load_submissions.clear() + @override def prepare_load( self, keys: Collection[OffloadKey], req_context: ReqContext ) -> LoadStoreSpec: @@ -362,6 +366,7 @@ class TieringOffloadingManager(OffloadingManager): return self.primary_tier.prepare_load(keys, req_context) + @override def touch(self, keys: Collection[OffloadKey], req_context: ReqContext): """ Mark blocks as recently used in all tiers. @@ -374,6 +379,7 @@ class TieringOffloadingManager(OffloadingManager): for tier in self.secondary_tiers: tier.touch(keys, req_context) + @override def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext): """ Mark blocks as done loading from primary tier to GPU. @@ -387,6 +393,7 @@ class TieringOffloadingManager(OffloadingManager): """ self.primary_tier.complete_load(keys, req_context) + @override def prepare_store( self, keys: Collection[OffloadKey], req_context: ReqContext ) -> PrepareStoreOutput | None: @@ -470,6 +477,7 @@ class TieringOffloadingManager(OffloadingManager): self._transfer_jobs[job_id] = job_metadata tier.submit_store(job_metadata) + @override def complete_store( self, keys: Collection[OffloadKey], @@ -528,6 +536,7 @@ class TieringOffloadingManager(OffloadingManager): # Note: The async transfers are now in flight. Their completion is # tracked via get_finished_jobs() / _maybe_process_finished_jobs(). + @override def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: """ Query each secondary tier for its offload policy preference. @@ -547,12 +556,14 @@ class TieringOffloadingManager(OffloadingManager): ) return RequestOffloadingContext(policy=policy) + @override def on_request_finished(self, req_context: ReqContext) -> None: self.primary_tier.on_request_finished(req_context) for tier in self.secondary_tiers: tier.on_request_finished(req_context) self._request_level_tiers.pop(req_context.req_id, None) + @override def take_events(self) -> Iterable[OffloadingEvent]: """ End-of-step hook: flush deferred work, yield events, reset per-step state. @@ -585,6 +596,7 @@ class TieringOffloadingManager(OffloadingManager): yield from self.primary_tier.take_events() + @override def shutdown(self) -> None: """Shutdown all tiers and release resources.""" for tier in self.secondary_tiers: