diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index b30c5d066a..dc9d95499c 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -7,7 +7,7 @@ Core abstractions for KV cache offloading in vLLM v1. from __future__ import annotations from abc import ABC, abstractmethod -from collections.abc import Iterable, Iterator, Sequence +from collections.abc import Collection, Iterable, Iterator, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Any, NewType @@ -128,7 +128,7 @@ class OffloadingManager(ABC): @abstractmethod def prepare_load( self, - keys: Sequence[OffloadKey], + keys: Collection[OffloadKey], req_context: ReqContext, ) -> LoadStoreSpec: """ @@ -147,7 +147,7 @@ class OffloadingManager(ABC): """ pass - def touch(self, keys: Sequence[OffloadKey]): + def touch(self, keys: Collection[OffloadKey]): """ Mark the given blocks as recently used. This could in practice mean moving them to the end of an LRU list. @@ -157,7 +157,7 @@ class OffloadingManager(ABC): """ return - def complete_load(self, keys: Iterable[OffloadKey]): + def complete_load(self, keys: Collection[OffloadKey]): """ Marks previous blocks that were prepared to load as done loading. @@ -169,7 +169,7 @@ class OffloadingManager(ABC): @abstractmethod def prepare_store( self, - keys: Sequence[OffloadKey], + keys: Collection[OffloadKey], req_context: ReqContext, ) -> PrepareStoreOutput | None: """ @@ -189,7 +189,7 @@ class OffloadingManager(ABC): """ pass - def complete_store(self, keys: Iterable[OffloadKey], success: bool = True): + def complete_store(self, keys: Collection[OffloadKey], success: bool = True): """ Marks blocks which were previously prepared to be stored, as stored. Following this call, the blocks become loadable. diff --git a/vllm/v1/kv_offload/cpu/manager.py b/vllm/v1/kv_offload/cpu/manager.py index 3527773a20..90d56915cc 100644 --- a/vllm/v1/kv_offload/cpu/manager.py +++ b/vllm/v1/kv_offload/cpu/manager.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Iterable, Sequence +from collections.abc import Collection, Iterable from typing import Literal from vllm.v1.kv_offload.base import ( @@ -90,7 +90,7 @@ class CPUOffloadingManager(OffloadingManager): def prepare_load( self, - keys: Sequence[OffloadKey], + keys: Collection[OffloadKey], req_context: ReqContext, ) -> LoadStoreSpec: blocks = [] @@ -102,10 +102,10 @@ class CPUOffloadingManager(OffloadingManager): blocks.append(block) return self._get_load_store_spec(keys, blocks) - def touch(self, keys: Sequence[OffloadKey]) -> None: + def touch(self, keys: Collection[OffloadKey]) -> None: self._policy.touch(keys) - def complete_load(self, keys: Iterable[OffloadKey]) -> None: + def complete_load(self, keys: Collection[OffloadKey]) -> None: for key in keys: block = self._policy.get(key) assert block is not None, f"Block {key!r} not found" @@ -114,7 +114,7 @@ class CPUOffloadingManager(OffloadingManager): def prepare_store( self, - keys: Sequence[OffloadKey], + keys: Collection[OffloadKey], req_context: ReqContext, ) -> PrepareStoreOutput | None: # filter out blocks that are already stored @@ -167,7 +167,9 @@ class CPUOffloadingManager(OffloadingManager): evicted_keys=to_evict, ) - def complete_store(self, keys: Iterable[OffloadKey], success: bool = True) -> None: + def complete_store( + self, keys: Collection[OffloadKey], success: bool = True + ) -> None: stored_keys: list[OffloadKey] = [] if success: diff --git a/vllm/v1/kv_offload/reuse_manager.py b/vllm/v1/kv_offload/reuse_manager.py index 2e556ca8d0..6cb0a5f759 100644 --- a/vllm/v1/kv_offload/reuse_manager.py +++ b/vllm/v1/kv_offload/reuse_manager.py @@ -8,7 +8,7 @@ FilterReusedOffloadingManager — OffloadingManager decorator that skips """ from collections import OrderedDict -from collections.abc import Iterable, Sequence +from collections.abc import Collection, Iterable from vllm.v1.kv_offload.base import ( LoadStoreSpec, @@ -79,7 +79,7 @@ class FilterReusedOffloadingManager(OffloadingManager): return self._backing.lookup(key, req_context) def prepare_store( - self, keys: Sequence[OffloadKey], req_context: ReqContext + self, keys: Collection[OffloadKey], req_context: ReqContext ) -> PrepareStoreOutput | None: """Filter out blocks below threshold, then delegate to backing. @@ -101,17 +101,19 @@ class FilterReusedOffloadingManager(OffloadingManager): # ------------------------------------------------------------------ def prepare_load( - self, keys: Sequence[OffloadKey], req_context: ReqContext + self, keys: Collection[OffloadKey], req_context: ReqContext ) -> LoadStoreSpec: return self._backing.prepare_load(keys, req_context) - def touch(self, keys: Sequence[OffloadKey]) -> None: + def touch(self, keys: Collection[OffloadKey]) -> None: return self._backing.touch(keys) - def complete_load(self, keys: Iterable[OffloadKey]) -> None: + def complete_load(self, keys: Collection[OffloadKey]) -> None: return self._backing.complete_load(keys) - def complete_store(self, keys: Iterable[OffloadKey], success: bool = True) -> None: + def complete_store( + self, keys: Collection[OffloadKey], success: bool = True + ) -> None: return self._backing.complete_store(keys, success) def take_events(self) -> Iterable[OffloadingEvent]: