From 3958a420f0ec29a2fe2e760a42418e61c1f0f5f7 Mon Sep 17 00:00:00 2001 From: Hotragn Pettugani <103170876+Hotragn@users.noreply.github.com> Date: Sat, 29 Aug 2026 01:08:01 -0400 Subject: [PATCH] [Bugfix][V1] Keep an encoder cache entry until its last occurrence is freed (#54284) Signed-off-by: hotragn --- tests/v1/core/test_encoder_cache_manager.py | 37 +++++++++++++++++++++ vllm/v1/core/encoder_cache_manager.py | 11 ++++++ 2 files changed, 48 insertions(+) diff --git a/tests/v1/core/test_encoder_cache_manager.py b/tests/v1/core/test_encoder_cache_manager.py index e56bcbf5c6..331a31ce66 100644 --- a/tests/v1/core/test_encoder_cache_manager.py +++ b/tests/v1/core/test_encoder_cache_manager.py @@ -349,6 +349,43 @@ def test_free_request_with_duplicate_mm_hashes(): assert "r1" not in manager.request_cached_ids +def test_duplicate_mm_hash_stays_referenced_until_last_free(): + """`cached` holds one reference per request, not per position, so freeing + the first of two occurrences must not release the entry.""" + manager = EncoderCacheManager(cache_size=20) + req = MockRequest("r1", ["imgA", "imgA"], [4, 4]) + + manager.allocate(req, 0) + assert manager.check_and_update_cache(req, 1) + + manager.free_encoder_input(req, 0) + assert manager.cached["imgA"] == {"r1"} + assert "imgA" not in manager.freeable + assert manager.num_freeable_slots == 16 + + manager.free_encoder_input(req, 1) + assert not manager.cached["imgA"] + assert manager.freeable["imgA"] == 4 + assert manager.num_freeable_slots == 20 + + +def test_duplicate_mm_hash_is_not_evicted_before_last_use(): + """An item repeated within one request (an image carried across + conversation turns) must not become reclaimable capacity while a later + occurrence still has to be spliced in, or the encoder recomputes it.""" + manager = EncoderCacheManager(cache_size=8) + req = MockRequest("r1", ["imgA", "imgA"], [4, 4]) + other = MockRequest("r2", ["imgB"], [8]) + + manager.allocate(req, 0) + assert manager.check_and_update_cache(req, 1) + manager.free_encoder_input(req, 0) + + assert not manager.can_allocate(other, 0, int(1e9), 0) + assert manager.check_and_update_cache(req, 1) + assert manager.get_freed_mm_hashes() == [] + + def test_encoder_decoder_cache_manager_reset(): manager = EncoderDecoderCacheManager(cache_size=20) diff --git a/vllm/v1/core/encoder_cache_manager.py b/vllm/v1/core/encoder_cache_manager.py index 8d2a81a11b..a3ed0c7aa1 100644 --- a/vllm/v1/core/encoder_cache_manager.py +++ b/vllm/v1/core/encoder_cache_manager.py @@ -242,6 +242,17 @@ class EncoderCacheManager: # The mm_hash not in cache or the req_id set is empty if not self.cached.get(mm_hash, None): return + # `cached` counts referencing requests, not positions, so one request + # that repeats an item (an image carried across conversation turns) has + # a single reference covering every occurrence. Hold it until the last + # occurrence is freed: dropping it at the first makes the entry + # evictable while the request still needs it, and the encoder then + # recomputes an item it already has. + if any( + request.mm_features[other_id].identifier == mm_hash + for other_id in self.request_cached_ids.get(req_id, ()) + ): + return self.cached[mm_hash].discard(req_id) if not self.cached[mm_hash]: num_encoder_embeds = request.get_num_encoder_embeds(input_id)