OffloadingConnector: Prevent redundant loads (#29087)

Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Or Ozeri
2026-01-21 01:15:42 +00:00
committed by GitHub
parent c78ee240b3
commit 7013e9ac8f
5 changed files with 109 additions and 8 deletions
+4 -2
View File
@@ -68,7 +68,7 @@ class OffloadingEvent:
class OffloadingManager(ABC):
@abstractmethod
def lookup(self, block_hashes: Iterable[BlockHash]) -> int:
def lookup(self, block_hashes: Iterable[BlockHash]) -> int | None:
"""
Finds the length of the maximal series of blocks, starting from the
first one, that are all offloaded.
@@ -78,7 +78,9 @@ class OffloadingManager(ABC):
Returns:
An integer representing the maximal number of blocks that
are currently offloaded.
are currently offloaded, or None if the lookup should be retried
later. Returning None will delay the request handling by the vLLM
scheduler.
"""
pass
+1 -1
View File
@@ -63,7 +63,7 @@ class ARCOffloadingManager(OffloadingManager):
self.events: list[OffloadingEvent] | None = [] if enable_events else None
self.cache_capacity: int = self.backend.get_num_free_blocks()
def lookup(self, block_hashes: Iterable[BlockHash]) -> int:
def lookup(self, block_hashes: Iterable[BlockHash]) -> int | None:
hit_count = 0
for block_hash in block_hashes:
block = self.t1.get(block_hash) or self.t2.get(block_hash)
+1 -1
View File
@@ -24,7 +24,7 @@ class LRUOffloadingManager(OffloadingManager):
self.blocks: OrderedDict[BlockHash, BlockStatus] = OrderedDict()
self.events: list[OffloadingEvent] | None = [] if enable_events else None
def lookup(self, block_hashes: Iterable[BlockHash]) -> int:
def lookup(self, block_hashes: Iterable[BlockHash]) -> int | None:
hit_count = 0
for block_hash in block_hashes:
block = self.blocks.get(block_hash)