[KV Offload] Use Collection instead of Sequence/Iterable for OffloadingManager key parameters (#41361)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
@@ -7,7 +7,7 @@ Core abstractions for KV cache offloading in vLLM v1.
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterable, Iterator, Sequence
|
||||
from collections.abc import Collection, Iterable, Iterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, NewType
|
||||
|
||||
@@ -128,7 +128,7 @@ class OffloadingManager(ABC):
|
||||
@abstractmethod
|
||||
def prepare_load(
|
||||
self,
|
||||
keys: Sequence[OffloadKey],
|
||||
keys: Collection[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
) -> LoadStoreSpec:
|
||||
"""
|
||||
@@ -147,7 +147,7 @@ class OffloadingManager(ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
def touch(self, keys: Sequence[OffloadKey]):
|
||||
def touch(self, keys: Collection[OffloadKey]):
|
||||
"""
|
||||
Mark the given blocks as recently used.
|
||||
This could in practice mean moving them to the end of an LRU list.
|
||||
@@ -157,7 +157,7 @@ class OffloadingManager(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
def complete_load(self, keys: Iterable[OffloadKey]):
|
||||
def complete_load(self, keys: Collection[OffloadKey]):
|
||||
"""
|
||||
Marks previous blocks that were prepared to load as done loading.
|
||||
|
||||
@@ -169,7 +169,7 @@ class OffloadingManager(ABC):
|
||||
@abstractmethod
|
||||
def prepare_store(
|
||||
self,
|
||||
keys: Sequence[OffloadKey],
|
||||
keys: Collection[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
) -> PrepareStoreOutput | None:
|
||||
"""
|
||||
@@ -189,7 +189,7 @@ class OffloadingManager(ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
def complete_store(self, keys: Iterable[OffloadKey], success: bool = True):
|
||||
def complete_store(self, keys: Collection[OffloadKey], success: bool = True):
|
||||
"""
|
||||
Marks blocks which were previously prepared to be stored, as stored.
|
||||
Following this call, the blocks become loadable.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable, Sequence
|
||||
from collections.abc import Collection, Iterable
|
||||
from typing import Literal
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
@@ -90,7 +90,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
|
||||
def prepare_load(
|
||||
self,
|
||||
keys: Sequence[OffloadKey],
|
||||
keys: Collection[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
) -> LoadStoreSpec:
|
||||
blocks = []
|
||||
@@ -102,10 +102,10 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
blocks.append(block)
|
||||
return self._get_load_store_spec(keys, blocks)
|
||||
|
||||
def touch(self, keys: Sequence[OffloadKey]) -> None:
|
||||
def touch(self, keys: Collection[OffloadKey]) -> None:
|
||||
self._policy.touch(keys)
|
||||
|
||||
def complete_load(self, keys: Iterable[OffloadKey]) -> None:
|
||||
def complete_load(self, keys: Collection[OffloadKey]) -> None:
|
||||
for key in keys:
|
||||
block = self._policy.get(key)
|
||||
assert block is not None, f"Block {key!r} not found"
|
||||
@@ -114,7 +114,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
|
||||
def prepare_store(
|
||||
self,
|
||||
keys: Sequence[OffloadKey],
|
||||
keys: Collection[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
) -> PrepareStoreOutput | None:
|
||||
# filter out blocks that are already stored
|
||||
@@ -167,7 +167,9 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
evicted_keys=to_evict,
|
||||
)
|
||||
|
||||
def complete_store(self, keys: Iterable[OffloadKey], success: bool = True) -> None:
|
||||
def complete_store(
|
||||
self, keys: Collection[OffloadKey], success: bool = True
|
||||
) -> None:
|
||||
stored_keys: list[OffloadKey] = []
|
||||
|
||||
if success:
|
||||
|
||||
@@ -8,7 +8,7 @@ FilterReusedOffloadingManager — OffloadingManager decorator that skips
|
||||
"""
|
||||
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Iterable, Sequence
|
||||
from collections.abc import Collection, Iterable
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LoadStoreSpec,
|
||||
@@ -79,7 +79,7 @@ class FilterReusedOffloadingManager(OffloadingManager):
|
||||
return self._backing.lookup(key, req_context)
|
||||
|
||||
def prepare_store(
|
||||
self, keys: Sequence[OffloadKey], req_context: ReqContext
|
||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||
) -> PrepareStoreOutput | None:
|
||||
"""Filter out blocks below threshold, then delegate to backing.
|
||||
|
||||
@@ -101,17 +101,19 @@ class FilterReusedOffloadingManager(OffloadingManager):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def prepare_load(
|
||||
self, keys: Sequence[OffloadKey], req_context: ReqContext
|
||||
self, keys: Collection[OffloadKey], req_context: ReqContext
|
||||
) -> LoadStoreSpec:
|
||||
return self._backing.prepare_load(keys, req_context)
|
||||
|
||||
def touch(self, keys: Sequence[OffloadKey]) -> None:
|
||||
def touch(self, keys: Collection[OffloadKey]) -> None:
|
||||
return self._backing.touch(keys)
|
||||
|
||||
def complete_load(self, keys: Iterable[OffloadKey]) -> None:
|
||||
def complete_load(self, keys: Collection[OffloadKey]) -> None:
|
||||
return self._backing.complete_load(keys)
|
||||
|
||||
def complete_store(self, keys: Iterable[OffloadKey], success: bool = True) -> None:
|
||||
def complete_store(
|
||||
self, keys: Collection[OffloadKey], success: bool = True
|
||||
) -> None:
|
||||
return self._backing.complete_store(keys, success)
|
||||
|
||||
def take_events(self) -> Iterable[OffloadingEvent]:
|
||||
|
||||
Reference in New Issue
Block a user