[Bugfix][V1] Keep an encoder cache entry until its last occurrence is freed (#54284)
Signed-off-by: hotragn <hotragn.pettugani_2024@woxsen.edu.in>
This commit is contained in:
@@ -242,6 +242,17 @@ class EncoderCacheManager:
|
||||
# The mm_hash not in cache or the req_id set is empty
|
||||
if not self.cached.get(mm_hash, None):
|
||||
return
|
||||
# `cached` counts referencing requests, not positions, so one request
|
||||
# that repeats an item (an image carried across conversation turns) has
|
||||
# a single reference covering every occurrence. Hold it until the last
|
||||
# occurrence is freed: dropping it at the first makes the entry
|
||||
# evictable while the request still needs it, and the encoder then
|
||||
# recomputes an item it already has.
|
||||
if any(
|
||||
request.mm_features[other_id].identifier == mm_hash
|
||||
for other_id in self.request_cached_ids.get(req_id, ())
|
||||
):
|
||||
return
|
||||
self.cached[mm_hash].discard(req_id)
|
||||
if not self.cached[mm_hash]:
|
||||
num_encoder_embeds = request.get_num_encoder_embeds(input_id)
|
||||
|
||||
Reference in New Issue
Block a user