[V1][Scheduler] Apply Mamba alignment before encoder caps (#51603)

Signed-off-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
Co-authored-by: Akshat Anand <40275336+cipheraxat@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Jiangyun Zhu
2026-08-10 16:37:17 +08:00
committed by GitHub
co-authored by Akshat Anand Codex
parent d89ba6481b
commit 243c63baf5
2 changed files with 105 additions and 20 deletions
+83
View File
@@ -3612,6 +3612,89 @@ def test_scheduler_no_ec_connector_by_default():
assert scheduler.ec_connector is None
def test_mamba_align_encoder_cache_cap_makes_progress():
"""Two individually cacheable images must not deadlock Mamba alignment."""
block_size = 768
encoder_cache_size = 600
scheduler = create_scheduler(
max_num_batched_tokens=8192,
block_size=block_size,
enable_prefix_caching=True,
)
scheduler.need_mamba_block_aligned_split = True
scheduler.max_num_encoder_input_tokens = encoder_cache_size
scheduler.encoder_cache_manager = EncoderCacheManager(cache_size=encoder_cache_size)
first_image_end = 510
request = create_requests(
num_requests=1,
num_tokens=1010,
mm_positions=[
[
PlaceholderRange(offset=0, length=500),
PlaceholderRange(offset=first_image_end, length=500),
]
],
max_tokens=1,
block_size=block_size,
req_ids=["req"],
)[0]
scheduler.add_request(request)
output = scheduler.schedule()
assert output.num_scheduled_tokens[request.request_id] == first_image_end
assert output.scheduled_encoder_inputs[request.request_id] == [0]
scheduler.update_from_output(
output,
ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
output = scheduler.schedule()
next_block_boundary = block_size
assert output.num_scheduled_tokens[request.request_id] == (
next_block_boundary - first_image_end
)
assert output.scheduled_encoder_inputs[request.request_id] == [1]
def test_mamba_align_eagle_schedules_encoder_at_boundary():
"""EAGLE lookahead at an aligned MM boundary requires encoder cache."""
block_size = 512
scheduler = create_scheduler(
max_num_batched_tokens=700,
max_model_len=2048,
block_size=block_size,
enable_prefix_caching=True,
)
scheduler.need_mamba_block_aligned_split = True
scheduler.use_eagle = True
scheduler.max_num_encoder_input_tokens = 2048
scheduler.encoder_cache_manager = EncoderCacheManager(cache_size=2048)
request = create_requests(
num_requests=1,
num_tokens=1200,
mm_positions=[[PlaceholderRange(offset=block_size, length=100)]],
max_tokens=1,
block_size=block_size,
req_ids=["req"],
)[0]
scheduler.add_request(request)
output = scheduler.schedule()
assert output.num_scheduled_tokens[request.request_id] == block_size
assert output.scheduled_encoder_inputs[request.request_id] == [0]
@pytest.mark.parametrize("use_kv_connector", [False, True])
def test_ec_connector_text_only_request(use_kv_connector):
"""Test text-only requests don't allocate encoder cache."""
+22 -20
View File
@@ -530,6 +530,12 @@ class Scheduler(SchedulerInterface):
- self.num_sampled_tokens_per_step,
)
# Apply Mamba alignment before encoder caps.
if self.need_mamba_block_aligned_split:
num_new_tokens = self._mamba_block_aligned_split(
request, num_new_tokens
)
# Schedule encoder inputs.
encoder_inputs_to_schedule = None
external_load_encoder_input: list[int] = []
@@ -548,11 +554,6 @@ class Scheduler(SchedulerInterface):
shift_computed_tokens=1 if self.use_eagle else 0,
)
if self.need_mamba_block_aligned_split:
num_new_tokens = self._mamba_block_aligned_split(
request, num_new_tokens
)
if num_new_tokens == 0:
# The request cannot be scheduled because one of the following
# reasons:
@@ -906,6 +907,17 @@ class Scheduler(SchedulerInterface):
num_new_tokens = min(num_new_tokens, token_budget)
assert num_new_tokens > 0
# Apply Mamba alignment before encoder caps.
if self.need_mamba_block_aligned_split:
num_new_tokens = self._mamba_block_aligned_split(
request,
num_new_tokens,
num_new_local_computed_tokens,
num_external_computed_tokens,
)
if num_new_tokens == 0:
break
# Schedule encoder inputs.
if request.has_encoder_inputs:
(
@@ -924,17 +936,6 @@ class Scheduler(SchedulerInterface):
# The request cannot be scheduled.
break
# Skip block alignment when setting up async receive (no local work).
if self.need_mamba_block_aligned_split and not load_kv_async:
num_new_tokens = self._mamba_block_aligned_split(
request,
num_new_tokens,
num_new_local_computed_tokens,
num_external_computed_tokens,
)
if num_new_tokens == 0:
break
# During async KV load, no forward pass is run yet.
# Allocate speculative lookahead slots later to avoid
# mismatching local and remote block counts.
@@ -1501,10 +1502,13 @@ class Scheduler(SchedulerInterface):
mm_hashes_to_schedule = set()
num_embeds_to_schedule = 0
encoder_window_end = (
num_computed_tokens + num_new_tokens + shift_computed_tokens
)
lo, hi = get_mm_features_in_window(
mm_features,
start=num_computed_tokens,
end=num_computed_tokens + num_new_tokens + shift_computed_tokens,
end=encoder_window_end,
)
# For encoder-decoder, all inputs sit at start_pos=0, so lo=0 always.
if self.is_encoder_decoder:
@@ -1586,9 +1590,7 @@ class Scheduler(SchedulerInterface):
# Calculate the number of embeddings to schedule in the current range
# of scheduled encoder placeholder tokens.
start_idx_rel = max(0, num_computed_tokens - start_pos)
end_idx_rel = min(
num_encoder_tokens, num_computed_tokens + num_new_tokens - start_pos
)
end_idx_rel = min(num_encoder_tokens, encoder_window_end - start_pos)
curr_embeds_start, curr_embeds_end = (
mm_feature.mm_position.get_embeds_indices_in_range(
start_idx_rel, end_idx_rel