[Bugfix][V1] Keep an encoder cache entry until its last occurrence is freed (#54284)

Signed-off-by: hotragn <hotragn.pettugani_2024@woxsen.edu.in>
This commit is contained in:
Hotragn Pettugani
2026-08-29 05:08:01 +00:00
committed by GitHub
parent 6d4562c59b
commit 3958a420f0
2 changed files with 48 additions and 0 deletions
@@ -349,6 +349,43 @@ def test_free_request_with_duplicate_mm_hashes():
assert "r1" not in manager.request_cached_ids
def test_duplicate_mm_hash_stays_referenced_until_last_free():
"""`cached` holds one reference per request, not per position, so freeing
the first of two occurrences must not release the entry."""
manager = EncoderCacheManager(cache_size=20)
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
manager.allocate(req, 0)
assert manager.check_and_update_cache(req, 1)
manager.free_encoder_input(req, 0)
assert manager.cached["imgA"] == {"r1"}
assert "imgA" not in manager.freeable
assert manager.num_freeable_slots == 16
manager.free_encoder_input(req, 1)
assert not manager.cached["imgA"]
assert manager.freeable["imgA"] == 4
assert manager.num_freeable_slots == 20
def test_duplicate_mm_hash_is_not_evicted_before_last_use():
"""An item repeated within one request (an image carried across
conversation turns) must not become reclaimable capacity while a later
occurrence still has to be spliced in, or the encoder recomputes it."""
manager = EncoderCacheManager(cache_size=8)
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
other = MockRequest("r2", ["imgB"], [8])
manager.allocate(req, 0)
assert manager.check_and_update_cache(req, 1)
manager.free_encoder_input(req, 0)
assert not manager.can_allocate(other, 0, int(1e9), 0)
assert manager.check_and_update_cache(req, 1)
assert manager.get_freed_mm_hashes() == []
def test_encoder_decoder_cache_manager_reset():
manager = EncoderDecoderCacheManager(cache_size=20)
+11
View File
@@ -242,6 +242,17 @@ class EncoderCacheManager:
# The mm_hash not in cache or the req_id set is empty
if not self.cached.get(mm_hash, None):
return
# `cached` counts referencing requests, not positions, so one request
# that repeats an item (an image carried across conversation turns) has
# a single reference covering every occurrence. Hold it until the last
# occurrence is freed: dropping it at the first makes the entry
# evictable while the request still needs it, and the encoder then
# recomputes an item it already has.
if any(
request.mm_features[other_id].identifier == mm_hash
for other_id in self.request_cached_ids.get(req_id, ())
):
return
self.cached[mm_hash].discard(req_id)
if not self.cached[mm_hash]:
num_encoder_embeds = request.get_num_encoder_embeds(input_id)