[Perf] Improve multimodal item handling from O(n) to O(log n) per step (#44212)
Signed-off-by: Andy Lo <andy@mistral.ai>
This commit is contained in:
@@ -43,6 +43,7 @@ def test_basic_allocate_and_reuse():
|
||||
|
||||
assert cache.check_and_update_cache(req, 0)
|
||||
assert "r1" in cache.cached["imgA"]
|
||||
assert cache.request_cached_ids["r1"] == {0}
|
||||
assert cache.num_free_slots == 6
|
||||
|
||||
# Free twice to bring refcount to 0.
|
||||
@@ -50,6 +51,7 @@ def test_basic_allocate_and_reuse():
|
||||
cache.free_encoder_input(req, 0)
|
||||
|
||||
assert not cache.cached["imgA"]
|
||||
assert "r1" not in cache.request_cached_ids
|
||||
assert "imgA" in cache.freeable
|
||||
assert cache.num_freeable_slots == 10
|
||||
assert cache.num_free_slots == 6
|
||||
@@ -63,10 +65,12 @@ def test_freeing_decreases_refcount_and_moves_to_freeable():
|
||||
manager.allocate(req, 0)
|
||||
|
||||
assert len(manager.cached["img3"]) == 1
|
||||
assert manager.request_cached_ids["req2"] == {0}
|
||||
|
||||
manager.free_encoder_input(req, 0)
|
||||
|
||||
assert not manager.cached["img3"]
|
||||
assert "req2" not in manager.request_cached_ids
|
||||
assert "img3" in manager.freeable
|
||||
assert manager.num_freeable_slots == 10
|
||||
|
||||
@@ -83,11 +87,13 @@ def test_free_request_frees_all_inputs():
|
||||
|
||||
assert len(manager.cached["a"]) == 1
|
||||
assert len(manager.cached["b"]) == 1
|
||||
assert manager.request_cached_ids["req3"] == {0, 1}
|
||||
|
||||
manager.free(req)
|
||||
|
||||
assert not manager.cached["a"]
|
||||
assert not manager.cached["b"]
|
||||
assert "req3" not in manager.request_cached_ids
|
||||
assert "a" in manager.freeable
|
||||
assert "b" in manager.freeable
|
||||
assert manager.num_freeable_slots == 10
|
||||
@@ -108,6 +114,7 @@ def test_eviction_when_cache_is_full():
|
||||
|
||||
# 'x' should have been evicted.
|
||||
assert "x" not in manager.cached
|
||||
assert "req1" not in manager.request_cached_ids
|
||||
assert "x" in manager.get_freed_mm_hashes()
|
||||
|
||||
|
||||
@@ -137,6 +144,7 @@ def test_has_cache_restores_from_freeable():
|
||||
# Should restore from freeable.
|
||||
assert manager.check_and_update_cache(req, 0)
|
||||
assert len(manager.cached["imgZ"]) == 1
|
||||
assert manager.request_cached_ids["reqY"] == {0}
|
||||
assert "imgZ" not in manager.freeable
|
||||
assert manager.num_freeable_slots == 6
|
||||
|
||||
@@ -205,6 +213,7 @@ def test_encoder_cache_with_is_embed_mask():
|
||||
|
||||
assert manager.num_free_slots == 92
|
||||
assert "img1" in manager.cached
|
||||
assert manager.request_cached_ids["r1"] == {0}
|
||||
|
||||
old_size = 100
|
||||
new_size = request.mm_features[0].mm_position.get_num_embeds()
|
||||
@@ -276,6 +285,7 @@ def test_reset_clears_all_state():
|
||||
manager.reset()
|
||||
|
||||
assert len(manager.cached) == 0
|
||||
assert len(manager.request_cached_ids) == 0
|
||||
assert len(manager.freeable) == 0
|
||||
assert len(manager.freed) == 0
|
||||
assert manager.num_free_slots == 20
|
||||
@@ -298,6 +308,26 @@ def test_reset_allows_fresh_allocations():
|
||||
assert manager.num_free_slots == 2
|
||||
assert "img2" in manager.cached
|
||||
assert "img1" not in manager.cached
|
||||
assert manager.request_cached_ids["req2"] == {0}
|
||||
assert "req1" not in manager.request_cached_ids
|
||||
|
||||
|
||||
def test_free_request_with_duplicate_mm_hashes():
|
||||
"""Freeing a request whose two inputs share the same mm_hash must fully
|
||||
clean up request_cached_ids. After the first free_encoder_input call,
|
||||
cached[mm_hash] becomes empty; the second call must still remove the
|
||||
remaining input_id from request_cached_ids."""
|
||||
manager = EncoderCacheManager(cache_size=20)
|
||||
|
||||
req = MockRequest("r1", ["imgA", "imgA"], [4, 4])
|
||||
|
||||
manager.allocate(req, 0)
|
||||
# input 1 has the same hash, so it's already cached.
|
||||
assert manager.check_and_update_cache(req, 1)
|
||||
assert manager.request_cached_ids["r1"] == {0, 1}
|
||||
|
||||
manager.free(req)
|
||||
assert "r1" not in manager.request_cached_ids
|
||||
|
||||
|
||||
def test_encoder_decoder_cache_manager_reset():
|
||||
|
||||
Reference in New Issue
Block a user