[Bugfix][V1][Multimodal] Ignore stale same-step encoder cache evictions (#52482)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-08-16 22:21:55 +00:00
committed by GitHub
parent e3c1cb54fc
commit eee538d5da
2 changed files with 22 additions and 1 deletions
@@ -167,6 +167,25 @@ def test_get_freed_mm_hashes_clears_freed_list():
assert manager.get_freed_mm_hashes() == []
def test_reallocated_hash_is_not_reported_as_freed():
manager = EncoderCacheManager(cache_size=8)
req_a = MockRequest("reqA", ["a"], [4])
req_b = MockRequest("reqB", ["b"], [4])
req_c = MockRequest("reqC", ["c"], [4])
manager.allocate(req_a, 0)
manager.allocate(req_b, 0)
manager.free(req_a)
manager.free(req_b)
assert manager.can_allocate(req_c, 0, int(1e9), 0)
manager.allocate(req_c, 0)
assert manager.can_allocate(req_a, 0, int(1e9), 0)
manager.allocate(req_a, 0)
assert manager.get_freed_mm_hashes() == ["b"]
def test_schedule_request_multi_images_respect_space_limit():
manager = EncoderCacheManager(cache_size=10)
req = MockRequest("reqA", ["a", "b"], [5, 6])
+3 -1
View File
@@ -269,7 +269,9 @@ class EncoderCacheManager:
encoder outputs can be removed from their caches. The internal
list is cleared after this call.
"""
freed = self.freed
# An entry evicted early in the scheduling pass can be allocated again
# later in the same pass. Keep its worker-side tensor in that case.
freed = [mm_hash for mm_hash in self.freed if mm_hash not in self.cached]
self.freed = []
return freed