diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index f53ba2ad23..107a37dc91 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -741,7 +741,8 @@ def test_same_step_duplicate_encoder_input_stays_cached(): assert "image" not in cache.freeable -def test_no_mm_input_chunking(): +@pytest.mark.parametrize("warm_encoder_cache", [False, True]) +def test_no_mm_input_chunking(warm_encoder_cache: bool): # Disable multimodal input chunking. scheduler = create_scheduler( model="llava-hf/llava-1.5-7b-hf", @@ -753,6 +754,16 @@ def test_no_mm_input_chunking(): requests = create_requests( num_requests=1, num_tokens=1200, mm_positions=mm_positions ) + if warm_encoder_cache: + cached_request = create_requests( + num_requests=1, + num_tokens=1200, + mm_positions=mm_positions, + req_ids=["cache-primer"], + )[0] + scheduler.encoder_cache_manager.allocate(cached_request, 0) + scheduler.encoder_cache_manager.free(cached_request) + for request in requests: scheduler.add_request(request) @@ -780,6 +791,8 @@ def test_no_mm_input_chunking(): assert output.scheduled_cached_reqs.num_reqs == 1 assert len(output.finished_req_ids) == 0 assert output.num_scheduled_tokens[requests[0].request_id] == 800 + if warm_encoder_cache: + assert not output.scheduled_encoder_inputs # Test that we fail if we disable chunked mm input and use too small # of a max_num_batched_tokens for the mm input. @@ -791,6 +804,32 @@ def test_no_mm_input_chunking(): ) +def test_no_mm_input_chunking_for_same_step_duplicate(): + scheduler = create_scheduler( + model="llava-hf/llava-1.5-7b-hf", + max_num_batched_tokens=1024, + disable_chunked_mm_input=True, + max_model_len=2048, + ) + request = create_requests( + num_requests=1, + num_tokens=1352, + mm_hashes_list=[["image", "image"]], + mm_positions=[ + [ + PlaceholderRange(offset=100, length=576), + PlaceholderRange(offset=776, length=576), + ] + ], + )[0] + scheduler.add_request(request) + + output = scheduler.schedule() + + assert output.num_scheduled_tokens[request.request_id] == 776 + assert output.scheduled_encoder_inputs == {request.request_id: [0]} + + @pytest.mark.parametrize("enable_prefix_caching", [True, False]) def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool): """Test scheduling behavior with concurrent partial requests. diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index bcb6def938..cec96d2a49 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -1863,6 +1863,23 @@ class Scheduler(SchedulerInterface): # already calculated encoder inputs and can skip here. continue + # If no encoder input chunking is allowed, we do not want to + # partially schedule a multimodal item. If the scheduled range would + # only cover part of the mm input, roll back to before the mm item. + if ( + self.scheduler_config.disable_chunked_mm_input + and num_computed_tokens < start_pos + and (num_computed_tokens + num_new_tokens) + < (start_pos + num_encoder_tokens) + ): + # Account for EAGLE shift when rolling back to avoid + # encoder cache miss. This ensures the scheduled range + # stops before start_pos even with the shift. + num_new_tokens = max( + 0, start_pos - (num_computed_tokens + shift_computed_tokens) + ) + break + if not self.is_encoder_decoder: # We are not using the encoder cache for encoder-decoder models, # yet. @@ -1902,22 +1919,6 @@ class Scheduler(SchedulerInterface): # previous step. continue - # If no encoder input chunking is allowed, we do not want to - # partially schedule a multimodal item. If the scheduled range would - # only cover part of the mm input, roll back to before the mm item. - if ( - self.scheduler_config.disable_chunked_mm_input - and num_computed_tokens < start_pos - and (num_computed_tokens + num_new_tokens) - < (start_pos + num_encoder_tokens) - ): - # Account for EAGLE shift when rolling back to avoid - # encoder cache miss. This ensures the scheduled range - # stops before start_pos even with the shift. - num_new_tokens = max( - 0, start_pos - (num_computed_tokens + shift_computed_tokens) - ) - break if not self.encoder_cache_manager.can_allocate( request, i, encoder_compute_budget, num_embeds_to_schedule ):