[Bugfix][Core] Retain encoder cache references for repeated multimodal inputs (#59942)

Signed-off-by: bzsuni <bingzhe.sun@daocloud.io>
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
Co-authored-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
bzsuni
2026-10-05 15:52:46 +00:00
committed by GitHub
co-authored by cjackal
parent edde9d2ce4
commit 528772a4bf
2 changed files with 65 additions and 25 deletions
+27
View File
@@ -688,6 +688,33 @@ def test_throttle_capacity_bound_guard_admits():
assert "b" in output.num_scheduled_tokens
def test_same_step_duplicate_encoder_input_stays_cached():
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
max_num_batched_tokens=1024,
max_model_len=2048,
)
request = create_requests(
1,
num_tokens=2000,
req_ids=["repeated"],
mm_hashes_list=[["image", "image", "image"]],
mm_positions=[
[PlaceholderRange(offset=offset, length=576) for offset in (0, 600, 1300)]
],
)[0]
scheduler.add_request(request)
output = scheduler.schedule()
assert output.scheduled_encoder_inputs == {request.request_id: [0]}
_model_output(scheduler, output, [[]])
# The second occurrence is partially consumed; the third is not scheduled.
cache = scheduler.encoder_cache_manager
assert cache.get_cached_input_ids(request) == {1}
assert "image" not in cache.freeable
def test_no_mm_input_chunking():
# Disable multimodal input chunking.
scheduler = create_scheduler(