[Core] Simplify handling of scheduler_reserve_full_isl option (#41064)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-04-30 18:10:00 -07:00
committed by GitHub
parent a3c83ff2fd
commit dd5506a157
4 changed files with 30 additions and 58 deletions
+4 -2
View File
@@ -2568,7 +2568,9 @@ def test_can_fit_full_sequence_swa_cap_admits_long_prompt():
prompt_len = 32 * block_size
req = make_request("long", list(range(prompt_len)), block_size, sha256)
assert manager.can_fit_full_sequence(req)
assert (
manager.allocate_slots(req, block_size, full_sequence_must_fit=True) is not None
)
def test_can_fit_full_sequence_full_attention_still_gates_oversized():
@@ -2619,4 +2621,4 @@ def test_can_fit_full_sequence_full_attention_still_gates_oversized():
prompt_len = 16 * block_size
req = make_request("oversized", list(range(prompt_len)), block_size, sha256)
assert not manager.can_fit_full_sequence(req)
assert manager.allocate_slots(req, block_size, full_sequence_must_fit=True) is None
@@ -179,6 +179,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable):
num_external_computed_tokens: int = 0,
delay_cache_blocks: bool = False,
num_encoder_tokens: int = 0,
full_sequence_must_fit: bool = False,
):
ret = original_allocate_slots_fn(
self,
@@ -190,6 +191,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable):
num_external_computed_tokens,
delay_cache_blocks,
num_encoder_tokens,
full_sequence_must_fit,
)
if cur_step_action is not None:
cur_block_ids = self.coordinator.single_type_managers[0].req_to_blocks[
+23 -42
View File
@@ -222,46 +222,6 @@ class KVCacheManager:
return self.create_kv_cache_blocks(computed_blocks), num_new_computed_tokens
def can_fit_full_sequence(
self,
request: Request,
num_new_computed_tokens: int = 0,
new_computed_blocks: KVCacheBlocks | None = None,
num_external_computed_tokens: int = 0,
num_encoder_tokens: int = 0,
) -> bool:
"""Check if the KV cache has enough free blocks to hold the full
sequence, accounting for prefix cache hits and sliding window.
This is used as an admission gate to prevent over-admitting requests
when chunked prefill would otherwise only check the first chunk.
"""
if new_computed_blocks is not None:
new_computed_block_list = new_computed_blocks.blocks
else:
new_computed_block_list = self.empty_kv_cache_blocks.blocks
num_local_computed_tokens = (
request.num_computed_tokens + num_new_computed_tokens
)
total_computed_tokens = min(
num_local_computed_tokens + num_external_computed_tokens,
self.max_model_len,
)
full_num_tokens = min(request.num_tokens, self.max_model_len)
num_blocks_to_allocate = self.coordinator.get_num_blocks_to_allocate(
request_id=request.request_id,
num_tokens=full_num_tokens,
new_computed_blocks=new_computed_block_list,
num_encoder_tokens=num_encoder_tokens,
total_computed_tokens=total_computed_tokens,
num_tokens_main_model=full_num_tokens,
apply_admission_cap=True,
)
return num_blocks_to_allocate <= self.block_pool.get_num_free_blocks()
def allocate_slots(
self,
request: Request,
@@ -272,6 +232,7 @@ class KVCacheManager:
num_external_computed_tokens: int = 0,
delay_cache_blocks: bool = False,
num_encoder_tokens: int = 0,
full_sequence_must_fit: bool = False,
) -> KVCacheBlocks | None:
"""Add slots for a request with new tokens to append.
@@ -293,6 +254,10 @@ class KVCacheManager:
num_encoder_tokens: The number of encoder tokens to allocate for
cross-attention in encoder-decoder models(e.g., Whisper).
For decoder-only models, this should be 0.
full_sequence_must_fit: Only allocate blocks if the KV cache has enough
free blocks to hold the full sequence, accounting for prefix cache hits
and sliding window. Used as an admission gate to prevent over-admitting
requests when chunked prefill would otherwise only check the first chunk
Blocks layout:
```
@@ -366,10 +331,26 @@ class KVCacheManager:
num_local_computed_tokens + num_external_computed_tokens,
self.max_model_len,
)
if full_sequence_must_fit:
# First check and fail if the full request sequence won't fit.
full_num_tokens = min(request.num_tokens, self.max_model_len)
num_blocks_to_allocate = self.coordinator.get_num_blocks_to_allocate(
request_id=request.request_id,
num_tokens=full_num_tokens,
new_computed_blocks=new_computed_block_list,
num_encoder_tokens=num_encoder_tokens,
total_computed_tokens=total_computed_tokens,
num_tokens_main_model=full_num_tokens,
apply_admission_cap=True,
)
if num_blocks_to_allocate > self.block_pool.get_num_free_blocks():
return None
num_tokens_main_model = total_computed_tokens + num_new_tokens
num_tokens_need_slot = min(
num_tokens_main_model + num_lookahead_tokens,
self.max_model_len,
num_tokens_main_model + num_lookahead_tokens, self.max_model_len
)
# Free the blocks that are skipped during the attention computation
+1 -14
View File
@@ -741,20 +741,6 @@ class Scheduler(SchedulerInterface):
for i in encoder_inputs_to_schedule
)
if (
self.scheduler_reserve_full_isl
and not self.kv_cache_manager.can_fit_full_sequence(
request,
num_new_computed_tokens=num_new_local_computed_tokens,
new_computed_blocks=new_computed_blocks,
num_external_computed_tokens=num_external_computed_tokens,
num_encoder_tokens=num_encoder_tokens,
)
):
if request.has_encoder_inputs:
self.encoder_cache_manager.free(request)
break
new_blocks = self.kv_cache_manager.allocate_slots(
request,
num_new_tokens,
@@ -764,6 +750,7 @@ class Scheduler(SchedulerInterface):
num_external_computed_tokens=num_external_computed_tokens,
delay_cache_blocks=load_kv_async,
num_encoder_tokens=num_encoder_tokens,
full_sequence_must_fit=self.scheduler_reserve_full_isl,
)
if new_blocks is None: