[KV Offload] Use Collection instead of Sequence/Iterable for OffloadingManager key parameters (#41361)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
Ronen Schaffer
2026-05-01 05:18:38 +03:00
committed by GitHub
parent 7198940b39
commit 415a879899
3 changed files with 22 additions and 18 deletions
+6 -6
View File
@@ -7,7 +7,7 @@ Core abstractions for KV cache offloading in vLLM v1.
from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import Iterable, Iterator, Sequence
from collections.abc import Collection, Iterable, Iterator, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, NewType
@@ -128,7 +128,7 @@ class OffloadingManager(ABC):
@abstractmethod
def prepare_load(
self,
keys: Sequence[OffloadKey],
keys: Collection[OffloadKey],
req_context: ReqContext,
) -> LoadStoreSpec:
"""
@@ -147,7 +147,7 @@ class OffloadingManager(ABC):
"""
pass
def touch(self, keys: Sequence[OffloadKey]):
def touch(self, keys: Collection[OffloadKey]):
"""
Mark the given blocks as recently used.
This could in practice mean moving them to the end of an LRU list.
@@ -157,7 +157,7 @@ class OffloadingManager(ABC):
"""
return
def complete_load(self, keys: Iterable[OffloadKey]):
def complete_load(self, keys: Collection[OffloadKey]):
"""
Marks previous blocks that were prepared to load as done loading.
@@ -169,7 +169,7 @@ class OffloadingManager(ABC):
@abstractmethod
def prepare_store(
self,
keys: Sequence[OffloadKey],
keys: Collection[OffloadKey],
req_context: ReqContext,
) -> PrepareStoreOutput | None:
"""
@@ -189,7 +189,7 @@ class OffloadingManager(ABC):
"""
pass
def complete_store(self, keys: Iterable[OffloadKey], success: bool = True):
def complete_store(self, keys: Collection[OffloadKey], success: bool = True):
"""
Marks blocks which were previously prepared to be stored, as stored.
Following this call, the blocks become loadable.
+8 -6
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Sequence
from collections.abc import Collection, Iterable
from typing import Literal
from vllm.v1.kv_offload.base import (
@@ -90,7 +90,7 @@ class CPUOffloadingManager(OffloadingManager):
def prepare_load(
self,
keys: Sequence[OffloadKey],
keys: Collection[OffloadKey],
req_context: ReqContext,
) -> LoadStoreSpec:
blocks = []
@@ -102,10 +102,10 @@ class CPUOffloadingManager(OffloadingManager):
blocks.append(block)
return self._get_load_store_spec(keys, blocks)
def touch(self, keys: Sequence[OffloadKey]) -> None:
def touch(self, keys: Collection[OffloadKey]) -> None:
self._policy.touch(keys)
def complete_load(self, keys: Iterable[OffloadKey]) -> None:
def complete_load(self, keys: Collection[OffloadKey]) -> None:
for key in keys:
block = self._policy.get(key)
assert block is not None, f"Block {key!r} not found"
@@ -114,7 +114,7 @@ class CPUOffloadingManager(OffloadingManager):
def prepare_store(
self,
keys: Sequence[OffloadKey],
keys: Collection[OffloadKey],
req_context: ReqContext,
) -> PrepareStoreOutput | None:
# filter out blocks that are already stored
@@ -167,7 +167,9 @@ class CPUOffloadingManager(OffloadingManager):
evicted_keys=to_evict,
)
def complete_store(self, keys: Iterable[OffloadKey], success: bool = True) -> None:
def complete_store(
self, keys: Collection[OffloadKey], success: bool = True
) -> None:
stored_keys: list[OffloadKey] = []
if success:
+8 -6
View File
@@ -8,7 +8,7 @@ FilterReusedOffloadingManager — OffloadingManager decorator that skips
"""
from collections import OrderedDict
from collections.abc import Iterable, Sequence
from collections.abc import Collection, Iterable
from vllm.v1.kv_offload.base import (
LoadStoreSpec,
@@ -79,7 +79,7 @@ class FilterReusedOffloadingManager(OffloadingManager):
return self._backing.lookup(key, req_context)
def prepare_store(
self, keys: Sequence[OffloadKey], req_context: ReqContext
self, keys: Collection[OffloadKey], req_context: ReqContext
) -> PrepareStoreOutput | None:
"""Filter out blocks below threshold, then delegate to backing.
@@ -101,17 +101,19 @@ class FilterReusedOffloadingManager(OffloadingManager):
# ------------------------------------------------------------------
def prepare_load(
self, keys: Sequence[OffloadKey], req_context: ReqContext
self, keys: Collection[OffloadKey], req_context: ReqContext
) -> LoadStoreSpec:
return self._backing.prepare_load(keys, req_context)
def touch(self, keys: Sequence[OffloadKey]) -> None:
def touch(self, keys: Collection[OffloadKey]) -> None:
return self._backing.touch(keys)
def complete_load(self, keys: Iterable[OffloadKey]) -> None:
def complete_load(self, keys: Collection[OffloadKey]) -> None:
return self._backing.complete_load(keys)
def complete_store(self, keys: Iterable[OffloadKey], success: bool = True) -> None:
def complete_store(
self, keys: Collection[OffloadKey], success: bool = True
) -> None:
return self._backing.complete_store(keys, success)
def take_events(self) -> Iterable[OffloadingEvent]: