diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 78617aa1c1..c35c38911a 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -2568,7 +2568,9 @@ def test_can_fit_full_sequence_swa_cap_admits_long_prompt(): prompt_len = 32 * block_size req = make_request("long", list(range(prompt_len)), block_size, sha256) - assert manager.can_fit_full_sequence(req) + assert ( + manager.allocate_slots(req, block_size, full_sequence_must_fit=True) is not None + ) def test_can_fit_full_sequence_full_attention_still_gates_oversized(): @@ -2619,4 +2621,4 @@ def test_can_fit_full_sequence_full_attention_still_gates_oversized(): prompt_len = 16 * block_size req = make_request("oversized", list(range(prompt_len)), block_size, sha256) - assert not manager.can_fit_full_sequence(req) + assert manager.allocate_slots(req, block_size, full_sequence_must_fit=True) is None diff --git a/tests/v1/e2e/general/test_mamba_prefix_cache.py b/tests/v1/e2e/general/test_mamba_prefix_cache.py index 8b9f7bb6c5..6ec9e7656e 100644 --- a/tests/v1/e2e/general/test_mamba_prefix_cache.py +++ b/tests/v1/e2e/general/test_mamba_prefix_cache.py @@ -179,6 +179,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable): num_external_computed_tokens: int = 0, delay_cache_blocks: bool = False, num_encoder_tokens: int = 0, + full_sequence_must_fit: bool = False, ): ret = original_allocate_slots_fn( self, @@ -190,6 +191,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable): num_external_computed_tokens, delay_cache_blocks, num_encoder_tokens, + full_sequence_must_fit, ) if cur_step_action is not None: cur_block_ids = self.coordinator.single_type_managers[0].req_to_blocks[ diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index ad82e5a1d5..431776870c 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -222,46 +222,6 @@ class KVCacheManager: return self.create_kv_cache_blocks(computed_blocks), num_new_computed_tokens - def can_fit_full_sequence( - self, - request: Request, - num_new_computed_tokens: int = 0, - new_computed_blocks: KVCacheBlocks | None = None, - num_external_computed_tokens: int = 0, - num_encoder_tokens: int = 0, - ) -> bool: - """Check if the KV cache has enough free blocks to hold the full - sequence, accounting for prefix cache hits and sliding window. - - This is used as an admission gate to prevent over-admitting requests - when chunked prefill would otherwise only check the first chunk. - """ - if new_computed_blocks is not None: - new_computed_block_list = new_computed_blocks.blocks - else: - new_computed_block_list = self.empty_kv_cache_blocks.blocks - - num_local_computed_tokens = ( - request.num_computed_tokens + num_new_computed_tokens - ) - total_computed_tokens = min( - num_local_computed_tokens + num_external_computed_tokens, - self.max_model_len, - ) - full_num_tokens = min(request.num_tokens, self.max_model_len) - - num_blocks_to_allocate = self.coordinator.get_num_blocks_to_allocate( - request_id=request.request_id, - num_tokens=full_num_tokens, - new_computed_blocks=new_computed_block_list, - num_encoder_tokens=num_encoder_tokens, - total_computed_tokens=total_computed_tokens, - num_tokens_main_model=full_num_tokens, - apply_admission_cap=True, - ) - - return num_blocks_to_allocate <= self.block_pool.get_num_free_blocks() - def allocate_slots( self, request: Request, @@ -272,6 +232,7 @@ class KVCacheManager: num_external_computed_tokens: int = 0, delay_cache_blocks: bool = False, num_encoder_tokens: int = 0, + full_sequence_must_fit: bool = False, ) -> KVCacheBlocks | None: """Add slots for a request with new tokens to append. @@ -293,6 +254,10 @@ class KVCacheManager: num_encoder_tokens: The number of encoder tokens to allocate for cross-attention in encoder-decoder models(e.g., Whisper). For decoder-only models, this should be 0. + full_sequence_must_fit: Only allocate blocks if the KV cache has enough + free blocks to hold the full sequence, accounting for prefix cache hits + and sliding window. Used as an admission gate to prevent over-admitting + requests when chunked prefill would otherwise only check the first chunk Blocks layout: ``` @@ -366,10 +331,26 @@ class KVCacheManager: num_local_computed_tokens + num_external_computed_tokens, self.max_model_len, ) + + if full_sequence_must_fit: + # First check and fail if the full request sequence won't fit. + full_num_tokens = min(request.num_tokens, self.max_model_len) + + num_blocks_to_allocate = self.coordinator.get_num_blocks_to_allocate( + request_id=request.request_id, + num_tokens=full_num_tokens, + new_computed_blocks=new_computed_block_list, + num_encoder_tokens=num_encoder_tokens, + total_computed_tokens=total_computed_tokens, + num_tokens_main_model=full_num_tokens, + apply_admission_cap=True, + ) + if num_blocks_to_allocate > self.block_pool.get_num_free_blocks(): + return None + num_tokens_main_model = total_computed_tokens + num_new_tokens num_tokens_need_slot = min( - num_tokens_main_model + num_lookahead_tokens, - self.max_model_len, + num_tokens_main_model + num_lookahead_tokens, self.max_model_len ) # Free the blocks that are skipped during the attention computation diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 395fa80bfe..7820b858bd 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -741,20 +741,6 @@ class Scheduler(SchedulerInterface): for i in encoder_inputs_to_schedule ) - if ( - self.scheduler_reserve_full_isl - and not self.kv_cache_manager.can_fit_full_sequence( - request, - num_new_computed_tokens=num_new_local_computed_tokens, - new_computed_blocks=new_computed_blocks, - num_external_computed_tokens=num_external_computed_tokens, - num_encoder_tokens=num_encoder_tokens, - ) - ): - if request.has_encoder_inputs: - self.encoder_cache_manager.free(request) - break - new_blocks = self.kv_cache_manager.allocate_slots( request, num_new_tokens, @@ -764,6 +750,7 @@ class Scheduler(SchedulerInterface): num_external_computed_tokens=num_external_computed_tokens, delay_cache_blocks=load_kv_async, num_encoder_tokens=num_encoder_tokens, + full_sequence_must_fit=self.scheduler_reserve_full_isl, ) if new_blocks is None: