[Perf] Improve multimodal item handling from O(n) to O(log n) per step (#44212)

Signed-off-by: Andy Lo <andy@mistral.ai>
This commit is contained in:
Andy Lo
2026-06-03 11:00:26 +00:00
committed by GitHub
parent 1fa9ea09f6
commit 95b1615ec9
6 changed files with 92 additions and 44 deletions
@@ -43,6 +43,7 @@ def test_basic_allocate_and_reuse():
assert cache.check_and_update_cache(req, 0)
assert "r1" in cache.cached["imgA"]
assert cache.request_cached_ids["r1"] == {0}
assert cache.num_free_slots == 6
# Free twice to bring refcount to 0.
@@ -50,6 +51,7 @@ def test_basic_allocate_and_reuse():
cache.free_encoder_input(req, 0)
assert not cache.cached["imgA"]
assert "r1" not in cache.request_cached_ids
assert "imgA" in cache.freeable
assert cache.num_freeable_slots == 10
assert cache.num_free_slots == 6
@@ -63,10 +65,12 @@ def test_freeing_decreases_refcount_and_moves_to_freeable():
manager.allocate(req, 0)
assert len(manager.cached["img3"]) == 1
assert manager.request_cached_ids["req2"] == {0}
manager.free_encoder_input(req, 0)
assert not manager.cached["img3"]
assert "req2" not in manager.request_cached_ids
assert "img3" in manager.freeable
assert manager.num_freeable_slots == 10
@@ -83,11 +87,13 @@ def test_free_request_frees_all_inputs():
assert len(manager.cached["a"]) == 1
assert len(manager.cached["b"]) == 1
assert manager.request_cached_ids["req3"] == {0, 1}
manager.free(req)
assert not manager.cached["a"]
assert not manager.cached["b"]
assert "req3" not in manager.request_cached_ids
assert "a" in manager.freeable
assert "b" in manager.freeable
assert manager.num_freeable_slots == 10
@@ -108,6 +114,7 @@ def test_eviction_when_cache_is_full():
# 'x' should have been evicted.
assert "x" not in manager.cached
assert "req1" not in manager.request_cached_ids
assert "x" in manager.get_freed_mm_hashes()
@@ -137,6 +144,7 @@ def test_has_cache_restores_from_freeable():
# Should restore from freeable.
assert manager.check_and_update_cache(req, 0)
assert len(manager.cached["imgZ"]) == 1
assert manager.request_cached_ids["reqY"] == {0}
assert "imgZ" not in manager.freeable
assert manager.num_freeable_slots == 6
@@ -205,6 +213,7 @@ def test_encoder_cache_with_is_embed_mask():
assert manager.num_free_slots == 92
assert "img1" in manager.cached
assert manager.request_cached_ids["r1"] == {0}
old_size = 100
new_size = request.mm_features[0].mm_position.get_num_embeds()
@@ -276,6 +285,7 @@ def test_reset_clears_all_state():
manager.reset()
assert len(manager.cached) == 0
assert len(manager.request_cached_ids) == 0
assert len(manager.freeable) == 0
assert len(manager.freed) == 0
assert manager.num_free_slots == 20
@@ -298,6 +308,26 @@ def test_reset_allows_fresh_allocations():
assert manager.num_free_slots == 2
assert "img2" in manager.cached
assert "img1" not in manager.cached
assert manager.request_cached_ids["req2"] == {0}
assert "req1" not in manager.request_cached_ids
def test_free_request_with_duplicate_mm_hashes():
"""Freeing a request whose two inputs share the same mm_hash must fully
clean up request_cached_ids. After the first free_encoder_input call,
cached[mm_hash] becomes empty; the second call must still remove the
remaining input_id from request_cached_ids."""
manager = EncoderCacheManager(cache_size=20)
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
manager.allocate(req, 0)
# input 1 has the same hash, so it's already cached.
assert manager.check_and_update_cache(req, 1)
assert manager.request_cached_ids["r1"] == {0, 1}
manager.free(req)
assert "r1" not in manager.request_cached_ids
def test_encoder_decoder_cache_manager_reset():