[Bugfix][Core] Check multimodal chunk boundaries before encoder cache reuse (#59721)
Signed-off-by: ztysth <rnk.liulw@gmail.com> Signed-off-by: ztysth <166926442+ztysth@users.noreply.github.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
@@ -741,7 +741,8 @@ def test_same_step_duplicate_encoder_input_stays_cached():
|
||||
assert "image" not in cache.freeable
|
||||
|
||||
|
||||
def test_no_mm_input_chunking():
|
||||
@pytest.mark.parametrize("warm_encoder_cache", [False, True])
|
||||
def test_no_mm_input_chunking(warm_encoder_cache: bool):
|
||||
# Disable multimodal input chunking.
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
@@ -753,6 +754,16 @@ def test_no_mm_input_chunking():
|
||||
requests = create_requests(
|
||||
num_requests=1, num_tokens=1200, mm_positions=mm_positions
|
||||
)
|
||||
if warm_encoder_cache:
|
||||
cached_request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=1200,
|
||||
mm_positions=mm_positions,
|
||||
req_ids=["cache-primer"],
|
||||
)[0]
|
||||
scheduler.encoder_cache_manager.allocate(cached_request, 0)
|
||||
scheduler.encoder_cache_manager.free(cached_request)
|
||||
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
|
||||
@@ -780,6 +791,8 @@ def test_no_mm_input_chunking():
|
||||
assert output.scheduled_cached_reqs.num_reqs == 1
|
||||
assert len(output.finished_req_ids) == 0
|
||||
assert output.num_scheduled_tokens[requests[0].request_id] == 800
|
||||
if warm_encoder_cache:
|
||||
assert not output.scheduled_encoder_inputs
|
||||
|
||||
# Test that we fail if we disable chunked mm input and use too small
|
||||
# of a max_num_batched_tokens for the mm input.
|
||||
@@ -791,6 +804,32 @@ def test_no_mm_input_chunking():
|
||||
)
|
||||
|
||||
|
||||
def test_no_mm_input_chunking_for_same_step_duplicate():
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
max_num_batched_tokens=1024,
|
||||
disable_chunked_mm_input=True,
|
||||
max_model_len=2048,
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=1352,
|
||||
mm_hashes_list=[["image", "image"]],
|
||||
mm_positions=[
|
||||
[
|
||||
PlaceholderRange(offset=100, length=576),
|
||||
PlaceholderRange(offset=776, length=576),
|
||||
]
|
||||
],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert output.num_scheduled_tokens[request.request_id] == 776
|
||||
assert output.scheduled_encoder_inputs == {request.request_id: [0]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_prefix_caching", [True, False])
|
||||
def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool):
|
||||
"""Test scheduling behavior with concurrent partial requests.
|
||||
|
||||
Reference in New Issue
Block a user