[Bugfix][Core] Retain encoder cache references for repeated multimodal inputs (#59942)
Signed-off-by: bzsuni <bingzhe.sun@daocloud.io> Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com> Co-authored-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
@@ -688,6 +688,33 @@ def test_throttle_capacity_bound_guard_admits():
|
||||
assert "b" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def test_same_step_duplicate_encoder_input_stays_cached():
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
max_num_batched_tokens=1024,
|
||||
max_model_len=2048,
|
||||
)
|
||||
request = create_requests(
|
||||
1,
|
||||
num_tokens=2000,
|
||||
req_ids=["repeated"],
|
||||
mm_hashes_list=[["image", "image", "image"]],
|
||||
mm_positions=[
|
||||
[PlaceholderRange(offset=offset, length=576) for offset in (0, 600, 1300)]
|
||||
],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.scheduled_encoder_inputs == {request.request_id: [0]}
|
||||
_model_output(scheduler, output, [[]])
|
||||
|
||||
# The second occurrence is partially consumed; the third is not scheduled.
|
||||
cache = scheduler.encoder_cache_manager
|
||||
assert cache.get_cached_input_ids(request) == {1}
|
||||
assert "image" not in cache.freeable
|
||||
|
||||
|
||||
def test_no_mm_input_chunking():
|
||||
# Disable multimodal input chunking.
|
||||
scheduler = create_scheduler(
|
||||
|
||||
Reference in New Issue
Block a user