OffloadingConnector: Prevent redundant loads (#29087)
Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -68,7 +68,7 @@ class OffloadingEvent:
|
||||
|
||||
class OffloadingManager(ABC):
|
||||
@abstractmethod
|
||||
def lookup(self, block_hashes: Iterable[BlockHash]) -> int:
|
||||
def lookup(self, block_hashes: Iterable[BlockHash]) -> int | None:
|
||||
"""
|
||||
Finds the length of the maximal series of blocks, starting from the
|
||||
first one, that are all offloaded.
|
||||
@@ -78,7 +78,9 @@ class OffloadingManager(ABC):
|
||||
|
||||
Returns:
|
||||
An integer representing the maximal number of blocks that
|
||||
are currently offloaded.
|
||||
are currently offloaded, or None if the lookup should be retried
|
||||
later. Returning None will delay the request handling by the vLLM
|
||||
scheduler.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ class ARCOffloadingManager(OffloadingManager):
|
||||
self.events: list[OffloadingEvent] | None = [] if enable_events else None
|
||||
self.cache_capacity: int = self.backend.get_num_free_blocks()
|
||||
|
||||
def lookup(self, block_hashes: Iterable[BlockHash]) -> int:
|
||||
def lookup(self, block_hashes: Iterable[BlockHash]) -> int | None:
|
||||
hit_count = 0
|
||||
for block_hash in block_hashes:
|
||||
block = self.t1.get(block_hash) or self.t2.get(block_hash)
|
||||
|
||||
@@ -24,7 +24,7 @@ class LRUOffloadingManager(OffloadingManager):
|
||||
self.blocks: OrderedDict[BlockHash, BlockStatus] = OrderedDict()
|
||||
self.events: list[OffloadingEvent] | None = [] if enable_events else None
|
||||
|
||||
def lookup(self, block_hashes: Iterable[BlockHash]) -> int:
|
||||
def lookup(self, block_hashes: Iterable[BlockHash]) -> int | None:
|
||||
hit_count = 0
|
||||
for block_hash in block_hashes:
|
||||
block = self.blocks.get(block_hash)
|
||||
|
||||
Reference in New Issue
Block a user