[Bugfix][V1][Multimodal] Ignore stale same-step encoder cache evictions (#52482)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -167,6 +167,25 @@ def test_get_freed_mm_hashes_clears_freed_list():
|
||||
assert manager.get_freed_mm_hashes() == []
|
||||
|
||||
|
||||
def test_reallocated_hash_is_not_reported_as_freed():
|
||||
manager = EncoderCacheManager(cache_size=8)
|
||||
req_a = MockRequest("reqA", ["a"], [4])
|
||||
req_b = MockRequest("reqB", ["b"], [4])
|
||||
req_c = MockRequest("reqC", ["c"], [4])
|
||||
|
||||
manager.allocate(req_a, 0)
|
||||
manager.allocate(req_b, 0)
|
||||
manager.free(req_a)
|
||||
manager.free(req_b)
|
||||
|
||||
assert manager.can_allocate(req_c, 0, int(1e9), 0)
|
||||
manager.allocate(req_c, 0)
|
||||
assert manager.can_allocate(req_a, 0, int(1e9), 0)
|
||||
manager.allocate(req_a, 0)
|
||||
|
||||
assert manager.get_freed_mm_hashes() == ["b"]
|
||||
|
||||
|
||||
def test_schedule_request_multi_images_respect_space_limit():
|
||||
manager = EncoderCacheManager(cache_size=10)
|
||||
req = MockRequest("reqA", ["a", "b"], [5, 6])
|
||||
|
||||
@@ -269,7 +269,9 @@ class EncoderCacheManager:
|
||||
encoder outputs can be removed from their caches. The internal
|
||||
list is cleared after this call.
|
||||
"""
|
||||
freed = self.freed
|
||||
# An entry evicted early in the scheduling pass can be allocated again
|
||||
# later in the same pass. Keep its worker-side tensor in that case.
|
||||
freed = [mm_hash for mm_hash in self.freed if mm_hash not in self.cached]
|
||||
self.freed = []
|
||||
return freed
|
||||
|
||||
|
||||
Reference in New Issue
Block a user