From 243c63baf5239f961305e25160c1bf6d34096df4 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Mon, 10 Aug 2026 16:37:17 +0800 Subject: [PATCH] [V1][Scheduler] Apply Mamba alignment before encoder caps (#51603) Signed-off-by: Jiangyun Zhu Co-authored-by: Akshat Anand <40275336+cipheraxat@users.noreply.github.com> Co-authored-by: Codex --- tests/v1/core/test_scheduler.py | 83 +++++++++++++++++++++++++++++++++ vllm/v1/core/sched/scheduler.py | 42 +++++++++-------- 2 files changed, 105 insertions(+), 20 deletions(-) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index c419d50b5b..0e46f54f6f 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -3612,6 +3612,89 @@ def test_scheduler_no_ec_connector_by_default(): assert scheduler.ec_connector is None +def test_mamba_align_encoder_cache_cap_makes_progress(): + """Two individually cacheable images must not deadlock Mamba alignment.""" + block_size = 768 + encoder_cache_size = 600 + scheduler = create_scheduler( + max_num_batched_tokens=8192, + block_size=block_size, + enable_prefix_caching=True, + ) + scheduler.need_mamba_block_aligned_split = True + scheduler.max_num_encoder_input_tokens = encoder_cache_size + scheduler.encoder_cache_manager = EncoderCacheManager(cache_size=encoder_cache_size) + + first_image_end = 510 + request = create_requests( + num_requests=1, + num_tokens=1010, + mm_positions=[ + [ + PlaceholderRange(offset=0, length=500), + PlaceholderRange(offset=first_image_end, length=500), + ] + ], + max_tokens=1, + block_size=block_size, + req_ids=["req"], + )[0] + scheduler.add_request(request) + + output = scheduler.schedule() + assert output.num_scheduled_tokens[request.request_id] == first_image_end + assert output.scheduled_encoder_inputs[request.request_id] == [0] + + scheduler.update_from_output( + output, + ModelRunnerOutput( + req_ids=[request.request_id], + req_id_to_index={request.request_id: 0}, + sampled_token_ids=[[]], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[], + ), + ) + + output = scheduler.schedule() + next_block_boundary = block_size + assert output.num_scheduled_tokens[request.request_id] == ( + next_block_boundary - first_image_end + ) + assert output.scheduled_encoder_inputs[request.request_id] == [1] + + +def test_mamba_align_eagle_schedules_encoder_at_boundary(): + """EAGLE lookahead at an aligned MM boundary requires encoder cache.""" + block_size = 512 + scheduler = create_scheduler( + max_num_batched_tokens=700, + max_model_len=2048, + block_size=block_size, + enable_prefix_caching=True, + ) + scheduler.need_mamba_block_aligned_split = True + scheduler.use_eagle = True + scheduler.max_num_encoder_input_tokens = 2048 + scheduler.encoder_cache_manager = EncoderCacheManager(cache_size=2048) + + request = create_requests( + num_requests=1, + num_tokens=1200, + mm_positions=[[PlaceholderRange(offset=block_size, length=100)]], + max_tokens=1, + block_size=block_size, + req_ids=["req"], + )[0] + scheduler.add_request(request) + + output = scheduler.schedule() + + assert output.num_scheduled_tokens[request.request_id] == block_size + assert output.scheduled_encoder_inputs[request.request_id] == [0] + + @pytest.mark.parametrize("use_kv_connector", [False, True]) def test_ec_connector_text_only_request(use_kv_connector): """Test text-only requests don't allocate encoder cache.""" diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 15ef101799..c08a130213 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -530,6 +530,12 @@ class Scheduler(SchedulerInterface): - self.num_sampled_tokens_per_step, ) + # Apply Mamba alignment before encoder caps. + if self.need_mamba_block_aligned_split: + num_new_tokens = self._mamba_block_aligned_split( + request, num_new_tokens + ) + # Schedule encoder inputs. encoder_inputs_to_schedule = None external_load_encoder_input: list[int] = [] @@ -548,11 +554,6 @@ class Scheduler(SchedulerInterface): shift_computed_tokens=1 if self.use_eagle else 0, ) - if self.need_mamba_block_aligned_split: - num_new_tokens = self._mamba_block_aligned_split( - request, num_new_tokens - ) - if num_new_tokens == 0: # The request cannot be scheduled because one of the following # reasons: @@ -906,6 +907,17 @@ class Scheduler(SchedulerInterface): num_new_tokens = min(num_new_tokens, token_budget) assert num_new_tokens > 0 + # Apply Mamba alignment before encoder caps. + if self.need_mamba_block_aligned_split: + num_new_tokens = self._mamba_block_aligned_split( + request, + num_new_tokens, + num_new_local_computed_tokens, + num_external_computed_tokens, + ) + if num_new_tokens == 0: + break + # Schedule encoder inputs. if request.has_encoder_inputs: ( @@ -924,17 +936,6 @@ class Scheduler(SchedulerInterface): # The request cannot be scheduled. break - # Skip block alignment when setting up async receive (no local work). - if self.need_mamba_block_aligned_split and not load_kv_async: - num_new_tokens = self._mamba_block_aligned_split( - request, - num_new_tokens, - num_new_local_computed_tokens, - num_external_computed_tokens, - ) - if num_new_tokens == 0: - break - # During async KV load, no forward pass is run yet. # Allocate speculative lookahead slots later to avoid # mismatching local and remote block counts. @@ -1501,10 +1502,13 @@ class Scheduler(SchedulerInterface): mm_hashes_to_schedule = set() num_embeds_to_schedule = 0 + encoder_window_end = ( + num_computed_tokens + num_new_tokens + shift_computed_tokens + ) lo, hi = get_mm_features_in_window( mm_features, start=num_computed_tokens, - end=num_computed_tokens + num_new_tokens + shift_computed_tokens, + end=encoder_window_end, ) # For encoder-decoder, all inputs sit at start_pos=0, so lo=0 always. if self.is_encoder_decoder: @@ -1586,9 +1590,7 @@ class Scheduler(SchedulerInterface): # Calculate the number of embeddings to schedule in the current range # of scheduled encoder placeholder tokens. start_idx_rel = max(0, num_computed_tokens - start_pos) - end_idx_rel = min( - num_encoder_tokens, num_computed_tokens + num_new_tokens - start_pos - ) + end_idx_rel = min(num_encoder_tokens, encoder_window_end - start_pos) curr_embeds_start, curr_embeds_end = ( mm_feature.mm_position.get_embeds_indices_in_range( start_idx_rel, end_idx_rel