[V1][Scheduler] Apply Mamba alignment before encoder caps (#51603)
Signed-off-by: Jiangyun Zhu <riverclouds.zhu@qq.com> Co-authored-by: Akshat Anand <40275336+cipheraxat@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Akshat Anand
Codex
parent
d89ba6481b
commit
243c63baf5
@@ -3612,6 +3612,89 @@ def test_scheduler_no_ec_connector_by_default():
|
||||
assert scheduler.ec_connector is None
|
||||
|
||||
|
||||
def test_mamba_align_encoder_cache_cap_makes_progress():
|
||||
"""Two individually cacheable images must not deadlock Mamba alignment."""
|
||||
block_size = 768
|
||||
encoder_cache_size = 600
|
||||
scheduler = create_scheduler(
|
||||
max_num_batched_tokens=8192,
|
||||
block_size=block_size,
|
||||
enable_prefix_caching=True,
|
||||
)
|
||||
scheduler.need_mamba_block_aligned_split = True
|
||||
scheduler.max_num_encoder_input_tokens = encoder_cache_size
|
||||
scheduler.encoder_cache_manager = EncoderCacheManager(cache_size=encoder_cache_size)
|
||||
|
||||
first_image_end = 510
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=1010,
|
||||
mm_positions=[
|
||||
[
|
||||
PlaceholderRange(offset=0, length=500),
|
||||
PlaceholderRange(offset=first_image_end, length=500),
|
||||
]
|
||||
],
|
||||
max_tokens=1,
|
||||
block_size=block_size,
|
||||
req_ids=["req"],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.num_scheduled_tokens[request.request_id] == first_image_end
|
||||
assert output.scheduled_encoder_inputs[request.request_id] == [0]
|
||||
|
||||
scheduler.update_from_output(
|
||||
output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
output = scheduler.schedule()
|
||||
next_block_boundary = block_size
|
||||
assert output.num_scheduled_tokens[request.request_id] == (
|
||||
next_block_boundary - first_image_end
|
||||
)
|
||||
assert output.scheduled_encoder_inputs[request.request_id] == [1]
|
||||
|
||||
|
||||
def test_mamba_align_eagle_schedules_encoder_at_boundary():
|
||||
"""EAGLE lookahead at an aligned MM boundary requires encoder cache."""
|
||||
block_size = 512
|
||||
scheduler = create_scheduler(
|
||||
max_num_batched_tokens=700,
|
||||
max_model_len=2048,
|
||||
block_size=block_size,
|
||||
enable_prefix_caching=True,
|
||||
)
|
||||
scheduler.need_mamba_block_aligned_split = True
|
||||
scheduler.use_eagle = True
|
||||
scheduler.max_num_encoder_input_tokens = 2048
|
||||
scheduler.encoder_cache_manager = EncoderCacheManager(cache_size=2048)
|
||||
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=1200,
|
||||
mm_positions=[[PlaceholderRange(offset=block_size, length=100)]],
|
||||
max_tokens=1,
|
||||
block_size=block_size,
|
||||
req_ids=["req"],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert output.num_scheduled_tokens[request.request_id] == block_size
|
||||
assert output.scheduled_encoder_inputs[request.request_id] == [0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_kv_connector", [False, True])
|
||||
def test_ec_connector_text_only_request(use_kv_connector):
|
||||
"""Test text-only requests don't allocate encoder cache."""
|
||||
|
||||
@@ -530,6 +530,12 @@ class Scheduler(SchedulerInterface):
|
||||
- self.num_sampled_tokens_per_step,
|
||||
)
|
||||
|
||||
# Apply Mamba alignment before encoder caps.
|
||||
if self.need_mamba_block_aligned_split:
|
||||
num_new_tokens = self._mamba_block_aligned_split(
|
||||
request, num_new_tokens
|
||||
)
|
||||
|
||||
# Schedule encoder inputs.
|
||||
encoder_inputs_to_schedule = None
|
||||
external_load_encoder_input: list[int] = []
|
||||
@@ -548,11 +554,6 @@ class Scheduler(SchedulerInterface):
|
||||
shift_computed_tokens=1 if self.use_eagle else 0,
|
||||
)
|
||||
|
||||
if self.need_mamba_block_aligned_split:
|
||||
num_new_tokens = self._mamba_block_aligned_split(
|
||||
request, num_new_tokens
|
||||
)
|
||||
|
||||
if num_new_tokens == 0:
|
||||
# The request cannot be scheduled because one of the following
|
||||
# reasons:
|
||||
@@ -906,6 +907,17 @@ class Scheduler(SchedulerInterface):
|
||||
num_new_tokens = min(num_new_tokens, token_budget)
|
||||
assert num_new_tokens > 0
|
||||
|
||||
# Apply Mamba alignment before encoder caps.
|
||||
if self.need_mamba_block_aligned_split:
|
||||
num_new_tokens = self._mamba_block_aligned_split(
|
||||
request,
|
||||
num_new_tokens,
|
||||
num_new_local_computed_tokens,
|
||||
num_external_computed_tokens,
|
||||
)
|
||||
if num_new_tokens == 0:
|
||||
break
|
||||
|
||||
# Schedule encoder inputs.
|
||||
if request.has_encoder_inputs:
|
||||
(
|
||||
@@ -924,17 +936,6 @@ class Scheduler(SchedulerInterface):
|
||||
# The request cannot be scheduled.
|
||||
break
|
||||
|
||||
# Skip block alignment when setting up async receive (no local work).
|
||||
if self.need_mamba_block_aligned_split and not load_kv_async:
|
||||
num_new_tokens = self._mamba_block_aligned_split(
|
||||
request,
|
||||
num_new_tokens,
|
||||
num_new_local_computed_tokens,
|
||||
num_external_computed_tokens,
|
||||
)
|
||||
if num_new_tokens == 0:
|
||||
break
|
||||
|
||||
# During async KV load, no forward pass is run yet.
|
||||
# Allocate speculative lookahead slots later to avoid
|
||||
# mismatching local and remote block counts.
|
||||
@@ -1501,10 +1502,13 @@ class Scheduler(SchedulerInterface):
|
||||
mm_hashes_to_schedule = set()
|
||||
num_embeds_to_schedule = 0
|
||||
|
||||
encoder_window_end = (
|
||||
num_computed_tokens + num_new_tokens + shift_computed_tokens
|
||||
)
|
||||
lo, hi = get_mm_features_in_window(
|
||||
mm_features,
|
||||
start=num_computed_tokens,
|
||||
end=num_computed_tokens + num_new_tokens + shift_computed_tokens,
|
||||
end=encoder_window_end,
|
||||
)
|
||||
# For encoder-decoder, all inputs sit at start_pos=0, so lo=0 always.
|
||||
if self.is_encoder_decoder:
|
||||
@@ -1586,9 +1590,7 @@ class Scheduler(SchedulerInterface):
|
||||
# Calculate the number of embeddings to schedule in the current range
|
||||
# of scheduled encoder placeholder tokens.
|
||||
start_idx_rel = max(0, num_computed_tokens - start_pos)
|
||||
end_idx_rel = min(
|
||||
num_encoder_tokens, num_computed_tokens + num_new_tokens - start_pos
|
||||
)
|
||||
end_idx_rel = min(num_encoder_tokens, encoder_window_end - start_pos)
|
||||
curr_embeds_start, curr_embeds_end = (
|
||||
mm_feature.mm_position.get_embeds_indices_in_range(
|
||||
start_idx_rel, end_idx_rel
|
||||
|
||||
Reference in New Issue
Block a user