[Core] Simplify handling of scheduler_reserve_full_isl option (#41064)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -2568,7 +2568,9 @@ def test_can_fit_full_sequence_swa_cap_admits_long_prompt():
|
||||
prompt_len = 32 * block_size
|
||||
req = make_request("long", list(range(prompt_len)), block_size, sha256)
|
||||
|
||||
assert manager.can_fit_full_sequence(req)
|
||||
assert (
|
||||
manager.allocate_slots(req, block_size, full_sequence_must_fit=True) is not None
|
||||
)
|
||||
|
||||
|
||||
def test_can_fit_full_sequence_full_attention_still_gates_oversized():
|
||||
@@ -2619,4 +2621,4 @@ def test_can_fit_full_sequence_full_attention_still_gates_oversized():
|
||||
prompt_len = 16 * block_size
|
||||
req = make_request("oversized", list(range(prompt_len)), block_size, sha256)
|
||||
|
||||
assert not manager.can_fit_full_sequence(req)
|
||||
assert manager.allocate_slots(req, block_size, full_sequence_must_fit=True) is None
|
||||
|
||||
@@ -179,6 +179,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable):
|
||||
num_external_computed_tokens: int = 0,
|
||||
delay_cache_blocks: bool = False,
|
||||
num_encoder_tokens: int = 0,
|
||||
full_sequence_must_fit: bool = False,
|
||||
):
|
||||
ret = original_allocate_slots_fn(
|
||||
self,
|
||||
@@ -190,6 +191,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable):
|
||||
num_external_computed_tokens,
|
||||
delay_cache_blocks,
|
||||
num_encoder_tokens,
|
||||
full_sequence_must_fit,
|
||||
)
|
||||
if cur_step_action is not None:
|
||||
cur_block_ids = self.coordinator.single_type_managers[0].req_to_blocks[
|
||||
|
||||
@@ -222,46 +222,6 @@ class KVCacheManager:
|
||||
|
||||
return self.create_kv_cache_blocks(computed_blocks), num_new_computed_tokens
|
||||
|
||||
def can_fit_full_sequence(
|
||||
self,
|
||||
request: Request,
|
||||
num_new_computed_tokens: int = 0,
|
||||
new_computed_blocks: KVCacheBlocks | None = None,
|
||||
num_external_computed_tokens: int = 0,
|
||||
num_encoder_tokens: int = 0,
|
||||
) -> bool:
|
||||
"""Check if the KV cache has enough free blocks to hold the full
|
||||
sequence, accounting for prefix cache hits and sliding window.
|
||||
|
||||
This is used as an admission gate to prevent over-admitting requests
|
||||
when chunked prefill would otherwise only check the first chunk.
|
||||
"""
|
||||
if new_computed_blocks is not None:
|
||||
new_computed_block_list = new_computed_blocks.blocks
|
||||
else:
|
||||
new_computed_block_list = self.empty_kv_cache_blocks.blocks
|
||||
|
||||
num_local_computed_tokens = (
|
||||
request.num_computed_tokens + num_new_computed_tokens
|
||||
)
|
||||
total_computed_tokens = min(
|
||||
num_local_computed_tokens + num_external_computed_tokens,
|
||||
self.max_model_len,
|
||||
)
|
||||
full_num_tokens = min(request.num_tokens, self.max_model_len)
|
||||
|
||||
num_blocks_to_allocate = self.coordinator.get_num_blocks_to_allocate(
|
||||
request_id=request.request_id,
|
||||
num_tokens=full_num_tokens,
|
||||
new_computed_blocks=new_computed_block_list,
|
||||
num_encoder_tokens=num_encoder_tokens,
|
||||
total_computed_tokens=total_computed_tokens,
|
||||
num_tokens_main_model=full_num_tokens,
|
||||
apply_admission_cap=True,
|
||||
)
|
||||
|
||||
return num_blocks_to_allocate <= self.block_pool.get_num_free_blocks()
|
||||
|
||||
def allocate_slots(
|
||||
self,
|
||||
request: Request,
|
||||
@@ -272,6 +232,7 @@ class KVCacheManager:
|
||||
num_external_computed_tokens: int = 0,
|
||||
delay_cache_blocks: bool = False,
|
||||
num_encoder_tokens: int = 0,
|
||||
full_sequence_must_fit: bool = False,
|
||||
) -> KVCacheBlocks | None:
|
||||
"""Add slots for a request with new tokens to append.
|
||||
|
||||
@@ -293,6 +254,10 @@ class KVCacheManager:
|
||||
num_encoder_tokens: The number of encoder tokens to allocate for
|
||||
cross-attention in encoder-decoder models(e.g., Whisper).
|
||||
For decoder-only models, this should be 0.
|
||||
full_sequence_must_fit: Only allocate blocks if the KV cache has enough
|
||||
free blocks to hold the full sequence, accounting for prefix cache hits
|
||||
and sliding window. Used as an admission gate to prevent over-admitting
|
||||
requests when chunked prefill would otherwise only check the first chunk
|
||||
|
||||
Blocks layout:
|
||||
```
|
||||
@@ -366,10 +331,26 @@ class KVCacheManager:
|
||||
num_local_computed_tokens + num_external_computed_tokens,
|
||||
self.max_model_len,
|
||||
)
|
||||
|
||||
if full_sequence_must_fit:
|
||||
# First check and fail if the full request sequence won't fit.
|
||||
full_num_tokens = min(request.num_tokens, self.max_model_len)
|
||||
|
||||
num_blocks_to_allocate = self.coordinator.get_num_blocks_to_allocate(
|
||||
request_id=request.request_id,
|
||||
num_tokens=full_num_tokens,
|
||||
new_computed_blocks=new_computed_block_list,
|
||||
num_encoder_tokens=num_encoder_tokens,
|
||||
total_computed_tokens=total_computed_tokens,
|
||||
num_tokens_main_model=full_num_tokens,
|
||||
apply_admission_cap=True,
|
||||
)
|
||||
if num_blocks_to_allocate > self.block_pool.get_num_free_blocks():
|
||||
return None
|
||||
|
||||
num_tokens_main_model = total_computed_tokens + num_new_tokens
|
||||
num_tokens_need_slot = min(
|
||||
num_tokens_main_model + num_lookahead_tokens,
|
||||
self.max_model_len,
|
||||
num_tokens_main_model + num_lookahead_tokens, self.max_model_len
|
||||
)
|
||||
|
||||
# Free the blocks that are skipped during the attention computation
|
||||
|
||||
@@ -741,20 +741,6 @@ class Scheduler(SchedulerInterface):
|
||||
for i in encoder_inputs_to_schedule
|
||||
)
|
||||
|
||||
if (
|
||||
self.scheduler_reserve_full_isl
|
||||
and not self.kv_cache_manager.can_fit_full_sequence(
|
||||
request,
|
||||
num_new_computed_tokens=num_new_local_computed_tokens,
|
||||
new_computed_blocks=new_computed_blocks,
|
||||
num_external_computed_tokens=num_external_computed_tokens,
|
||||
num_encoder_tokens=num_encoder_tokens,
|
||||
)
|
||||
):
|
||||
if request.has_encoder_inputs:
|
||||
self.encoder_cache_manager.free(request)
|
||||
break
|
||||
|
||||
new_blocks = self.kv_cache_manager.allocate_slots(
|
||||
request,
|
||||
num_new_tokens,
|
||||
@@ -764,6 +750,7 @@ class Scheduler(SchedulerInterface):
|
||||
num_external_computed_tokens=num_external_computed_tokens,
|
||||
delay_cache_blocks=load_kv_async,
|
||||
num_encoder_tokens=num_encoder_tokens,
|
||||
full_sequence_must_fit=self.scheduler_reserve_full_isl,
|
||||
)
|
||||
|
||||
if new_blocks is None:
|
||||
|
||||
Reference in New Issue
Block a user