[V1][Scheduler] Apply Mamba alignment before encoder caps (#51603)

Signed-off-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
Co-authored-by: Akshat Anand <40275336+cipheraxat@users.noreply.github.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Jiangyun Zhu
2026-08-10 16:37:17 +08:00
committed by GitHub
co-authored by Akshat Anand Codex
parent d89ba6481b
commit 243c63baf5
2 changed files with 105 additions and 20 deletions
+83
View File
@@ -3612,6 +3612,89 @@ def test_scheduler_no_ec_connector_by_default():
assert scheduler.ec_connector is None
def test_mamba_align_encoder_cache_cap_makes_progress():
"""Two individually cacheable images must not deadlock Mamba alignment."""
block_size = 768
encoder_cache_size = 600
scheduler = create_scheduler(
max_num_batched_tokens=8192,
block_size=block_size,
enable_prefix_caching=True,
)
scheduler.need_mamba_block_aligned_split = True
scheduler.max_num_encoder_input_tokens = encoder_cache_size
scheduler.encoder_cache_manager = EncoderCacheManager(cache_size=encoder_cache_size)
first_image_end = 510
request = create_requests(
num_requests=1,
num_tokens=1010,
mm_positions=[
[
PlaceholderRange(offset=0, length=500),
PlaceholderRange(offset=first_image_end, length=500),
]
],
max_tokens=1,
block_size=block_size,
req_ids=["req"],
)[0]
scheduler.add_request(request)
output = scheduler.schedule()
assert output.num_scheduled_tokens[request.request_id] == first_image_end
assert output.scheduled_encoder_inputs[request.request_id] == [0]
scheduler.update_from_output(
output,
ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
output = scheduler.schedule()
next_block_boundary = block_size
assert output.num_scheduled_tokens[request.request_id] == (
next_block_boundary - first_image_end
)
assert output.scheduled_encoder_inputs[request.request_id] == [1]
def test_mamba_align_eagle_schedules_encoder_at_boundary():
"""EAGLE lookahead at an aligned MM boundary requires encoder cache."""
block_size = 512
scheduler = create_scheduler(
max_num_batched_tokens=700,
max_model_len=2048,
block_size=block_size,
enable_prefix_caching=True,
)
scheduler.need_mamba_block_aligned_split = True
scheduler.use_eagle = True
scheduler.max_num_encoder_input_tokens = 2048
scheduler.encoder_cache_manager = EncoderCacheManager(cache_size=2048)
request = create_requests(
num_requests=1,
num_tokens=1200,
mm_positions=[[PlaceholderRange(offset=block_size, length=100)]],
max_tokens=1,
block_size=block_size,
req_ids=["req"],
)[0]
scheduler.add_request(request)
output = scheduler.schedule()
assert output.num_scheduled_tokens[request.request_id] == block_size
assert output.scheduled_encoder_inputs[request.request_id] == [0]
@pytest.mark.parametrize("use_kv_connector", [False, True])
def test_ec_connector_text_only_request(use_kv_connector):
"""Test text-only requests don't allocate encoder cache."""