[Bugfix][Core] Check multimodal chunk boundaries before encoder cache reuse (#59721)

Signed-off-by: ztysth <rnk.liulw@gmail.com>
Signed-off-by: ztysth <166926442+ztysth@users.noreply.github.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
ztysth
2026-10-12 19:25:35 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 08582942d8
commit 1bdd3fa2d5
2 changed files with 57 additions and 17 deletions
+40 -1
View File
@@ -741,7 +741,8 @@ def test_same_step_duplicate_encoder_input_stays_cached():
assert "image" not in cache.freeable
def test_no_mm_input_chunking():
@pytest.mark.parametrize("warm_encoder_cache", [False, True])
def test_no_mm_input_chunking(warm_encoder_cache: bool):
# Disable multimodal input chunking.
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
@@ -753,6 +754,16 @@ def test_no_mm_input_chunking():
requests = create_requests(
num_requests=1, num_tokens=1200, mm_positions=mm_positions
)
if warm_encoder_cache:
cached_request = create_requests(
num_requests=1,
num_tokens=1200,
mm_positions=mm_positions,
req_ids=["cache-primer"],
)[0]
scheduler.encoder_cache_manager.allocate(cached_request, 0)
scheduler.encoder_cache_manager.free(cached_request)
for request in requests:
scheduler.add_request(request)
@@ -780,6 +791,8 @@ def test_no_mm_input_chunking():
assert output.scheduled_cached_reqs.num_reqs == 1
assert len(output.finished_req_ids) == 0
assert output.num_scheduled_tokens[requests[0].request_id] == 800
if warm_encoder_cache:
assert not output.scheduled_encoder_inputs
# Test that we fail if we disable chunked mm input and use too small
# of a max_num_batched_tokens for the mm input.
@@ -791,6 +804,32 @@ def test_no_mm_input_chunking():
)
def test_no_mm_input_chunking_for_same_step_duplicate():
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
max_num_batched_tokens=1024,
disable_chunked_mm_input=True,
max_model_len=2048,
)
request = create_requests(
num_requests=1,
num_tokens=1352,
mm_hashes_list=[["image", "image"]],
mm_positions=[
[
PlaceholderRange(offset=100, length=576),
PlaceholderRange(offset=776, length=576),
]
],
)[0]
scheduler.add_request(request)
output = scheduler.schedule()
assert output.num_scheduled_tokens[request.request_id] == 776
assert output.scheduled_encoder_inputs == {request.request_id: [0]}
@pytest.mark.parametrize("enable_prefix_caching", [True, False])
def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool):
"""Test scheduling behavior with concurrent partial requests.
+17 -16
View File
@@ -1863,6 +1863,23 @@ class Scheduler(SchedulerInterface):
# already calculated encoder inputs and can skip here.
continue
# If no encoder input chunking is allowed, we do not want to
# partially schedule a multimodal item. If the scheduled range would
# only cover part of the mm input, roll back to before the mm item.
if (
self.scheduler_config.disable_chunked_mm_input
and num_computed_tokens < start_pos
and (num_computed_tokens + num_new_tokens)
< (start_pos + num_encoder_tokens)
):
# Account for EAGLE shift when rolling back to avoid
# encoder cache miss. This ensures the scheduled range
# stops before start_pos even with the shift.
num_new_tokens = max(
0, start_pos - (num_computed_tokens + shift_computed_tokens)
)
break
if not self.is_encoder_decoder:
# We are not using the encoder cache for encoder-decoder models,
# yet.
@@ -1902,22 +1919,6 @@ class Scheduler(SchedulerInterface):
# previous step.
continue
# If no encoder input chunking is allowed, we do not want to
# partially schedule a multimodal item. If the scheduled range would
# only cover part of the mm input, roll back to before the mm item.
if (
self.scheduler_config.disable_chunked_mm_input
and num_computed_tokens < start_pos
and (num_computed_tokens + num_new_tokens)
< (start_pos + num_encoder_tokens)
):
# Account for EAGLE shift when rolling back to avoid
# encoder cache miss. This ensures the scheduled range
# stops before start_pos even with the shift.
num_new_tokens = max(
0, start_pos - (num_computed_tokens + shift_computed_tokens)
)
break
if not self.encoder_cache_manager.can_allocate(
request, i, encoder_compute_budget, num_embeds_to_schedule
):