[V1][Scheduler] Apply Mamba alignment before encoder caps (#51603)
Signed-off-by: Jiangyun Zhu <riverclouds.zhu@qq.com> Co-authored-by: Akshat Anand <40275336+cipheraxat@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Akshat Anand
Codex
parent
d89ba6481b
commit
243c63baf5
@@ -3612,6 +3612,89 @@ def test_scheduler_no_ec_connector_by_default():
|
||||
assert scheduler.ec_connector is None
|
||||
|
||||
|
||||
def test_mamba_align_encoder_cache_cap_makes_progress():
|
||||
"""Two individually cacheable images must not deadlock Mamba alignment."""
|
||||
block_size = 768
|
||||
encoder_cache_size = 600
|
||||
scheduler = create_scheduler(
|
||||
max_num_batched_tokens=8192,
|
||||
block_size=block_size,
|
||||
enable_prefix_caching=True,
|
||||
)
|
||||
scheduler.need_mamba_block_aligned_split = True
|
||||
scheduler.max_num_encoder_input_tokens = encoder_cache_size
|
||||
scheduler.encoder_cache_manager = EncoderCacheManager(cache_size=encoder_cache_size)
|
||||
|
||||
first_image_end = 510
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=1010,
|
||||
mm_positions=[
|
||||
[
|
||||
PlaceholderRange(offset=0, length=500),
|
||||
PlaceholderRange(offset=first_image_end, length=500),
|
||||
]
|
||||
],
|
||||
max_tokens=1,
|
||||
block_size=block_size,
|
||||
req_ids=["req"],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.num_scheduled_tokens[request.request_id] == first_image_end
|
||||
assert output.scheduled_encoder_inputs[request.request_id] == [0]
|
||||
|
||||
scheduler.update_from_output(
|
||||
output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
output = scheduler.schedule()
|
||||
next_block_boundary = block_size
|
||||
assert output.num_scheduled_tokens[request.request_id] == (
|
||||
next_block_boundary - first_image_end
|
||||
)
|
||||
assert output.scheduled_encoder_inputs[request.request_id] == [1]
|
||||
|
||||
|
||||
def test_mamba_align_eagle_schedules_encoder_at_boundary():
|
||||
"""EAGLE lookahead at an aligned MM boundary requires encoder cache."""
|
||||
block_size = 512
|
||||
scheduler = create_scheduler(
|
||||
max_num_batched_tokens=700,
|
||||
max_model_len=2048,
|
||||
block_size=block_size,
|
||||
enable_prefix_caching=True,
|
||||
)
|
||||
scheduler.need_mamba_block_aligned_split = True
|
||||
scheduler.use_eagle = True
|
||||
scheduler.max_num_encoder_input_tokens = 2048
|
||||
scheduler.encoder_cache_manager = EncoderCacheManager(cache_size=2048)
|
||||
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=1200,
|
||||
mm_positions=[[PlaceholderRange(offset=block_size, length=100)]],
|
||||
max_tokens=1,
|
||||
block_size=block_size,
|
||||
req_ids=["req"],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert output.num_scheduled_tokens[request.request_id] == block_size
|
||||
assert output.scheduled_encoder_inputs[request.request_id] == [0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_kv_connector", [False, True])
|
||||
def test_ec_connector_text_only_request(use_kv_connector):
|
||||
"""Test text-only requests don't allocate encoder cache."""
|
||||
|
||||
Reference in New Issue
Block a user