From 01abe975b5837f9e0722a196a28e57331735e0a6 Mon Sep 17 00:00:00 2001 From: bzsuni <86399306+bzsuni@users.noreply.github.com> Date: Sun, 11 Oct 2026 11:55:41 +0800 Subject: [PATCH] [Bugfix][KV Connector] Fix speculative block reservations across KV cache groups (#59880) Signed-off-by: bzsuni --- .../unit/test_remote_prefill_lifecycle.py | 133 ++++++++++++++---- tests/v1/kv_connector/unit/utils.py | 2 + vllm/v1/core/sched/scheduler.py | 34 ++--- 3 files changed, 128 insertions(+), 41 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py b/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py index 59fcc678bf..b72de811c0 100644 --- a/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py +++ b/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import copy +from dataclasses import replace from unittest.mock import patch import pytest @@ -761,35 +762,54 @@ def test_async_loads_both_admitted_when_pool_fits(): assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS -def test_async_load_reserves_blocks_for_promotion_margin(): - """An async load is not admitted unless the blocks its own promotion will - need are still free. +@pytest.mark.parametrize( + ("swa_enabled", "block_size", "num_spec_tokens", "num_blocks"), + [(False, 16, 3, 8), (True, 16, 3, 17), (True, 64, 31, 40)], + ids=["single-group", "multiple-groups", "different-block-sizes"], +) +def test_async_load_reserves_blocks_for_promotion_margin( + swa_enabled, block_size, num_spec_tokens, num_blocks +): + """Async loads must leave enough blocks for promotion in every cache group. - A parked load is allocated without lookahead slots, but promotion pads it - to ``1 + num_spec_tokens`` and asks for the lookahead margin on top. Without - reserving that margin, two loads can be admitted that together consume the - whole pool; the head then fails ``allocate_slots`` forever while the load - behind it is never reached, and with nothing running no block is ever freed. - - req_a (4 blocks) and req_b (3 blocks) exactly fill the 7 usable blocks, so - req_b must be held back in WAITING holding no blocks, leaving req_a room to - be promoted and run. + Otherwise two loads can fill the pool and prevent the head from being + promoted, with nothing running to free blocks and break the deadlock. """ - vllm_config = create_vllm_config(num_speculative_tokens=3) - BLOCK_SIZE = vllm_config.cache_config.block_size - scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7 + vllm_config = create_vllm_config( + block_size=block_size, + num_speculative_tokens=num_spec_tokens, + disable_hybrid_kv_cache_manager=False, + ) + hash_block_size = 16 + kv_cache_config = make_kv_cache_config( + hash_block_size, swa_enabled=swa_enabled, sw_size=4096, num_blocks=num_blocks + ) + full_group = kv_cache_config.kv_cache_groups[0] + full_group.kv_cache_spec = replace(full_group.kv_cache_spec, block_size=block_size) + if swa_enabled: + sliding_group = kv_cache_config.kv_cache_groups[-1] + sliding_group.kv_cache_spec = replace( + sliding_group.kv_cache_spec, + page_size_padded=full_group.kv_cache_spec.page_size_bytes, + ) + scheduler = create_scheduler( + vllm_config, + num_blocks=num_blocks, + kv_cache_config=kv_cache_config, + hash_block_size=hash_block_size, + ) req_a = create_request( request_id=1, - block_size=BLOCK_SIZE, - num_tokens=BLOCK_SIZE * 4, + block_size=hash_block_size, + num_tokens=block_size * 4, do_remote_prefill=True, max_tokens=1, ) req_b = create_request( request_id=2, - block_size=BLOCK_SIZE, - num_tokens=BLOCK_SIZE * 3, + block_size=hash_block_size, + num_tokens=block_size * 3, do_remote_prefill=True, max_tokens=1, ) @@ -800,16 +820,13 @@ def test_async_load_reserves_blocks_for_promotion_margin(): with patch.object( scheduler.connector, "get_num_new_matched_tokens", - side_effect=[(BLOCK_SIZE * 4, True), (BLOCK_SIZE * 3, True)], + side_effect=[(block_size * 4, True), (block_size * 3, True)], ): scheduler.schedule() assert req_a.status == RequestStatus.WAITING_FOR_REMOTE_KVS assert req_b.status == RequestStatus.WAITING - req_to_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[ - 0 - ].req_to_blocks - assert req_b.request_id not in req_to_blocks + assert not any(scheduler.kv_cache_manager.get_block_ids(req_b.request_id)) # req_a's load lands: it must be promotable and actually get scheduled. scheduler.update_from_output( @@ -821,6 +838,74 @@ def test_async_load_reserves_blocks_for_promotion_margin(): assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0 +@pytest.mark.parametrize( + ("mamba_cache_mode", "mamba_block_size", "num_blocks"), + [("none", 128, 18), ("align", 16, 18)], +) +def test_async_load_does_not_double_reserve_mamba_spec_blocks( + mamba_cache_mode, mamba_block_size, num_blocks +): + """Two hybrid loads fit when Mamba's speculative states are counted once.""" + block_size = 16 + num_spec_tokens = 3 + vllm_config = create_vllm_config( + block_size=mamba_block_size, + max_model_len=128, + num_speculative_tokens=num_spec_tokens, + disable_hybrid_kv_cache_manager=False, + ) + vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode + kv_cache_config = make_kv_cache_config( + block_size, + mamba_enabled=True, + mamba_cache_mode=mamba_cache_mode, + num_blocks=num_blocks, + ) + mamba_group = kv_cache_config.kv_cache_groups[-1] + mamba_group.kv_cache_spec = replace( + mamba_group.kv_cache_spec, + block_size=mamba_block_size, + num_speculative_blocks=num_spec_tokens, + ) + scheduler = create_scheduler( + vllm_config, + num_blocks=num_blocks, + kv_cache_config=kv_cache_config, + hash_block_size=block_size, + ) + reqs = [ + create_request( + request_id=i, + block_size=block_size, + num_tokens=num_tokens, + do_remote_prefill=True, + max_tokens=1, + ) + for i, num_tokens in enumerate((64, 48), 1) + ] + for req in reqs: + scheduler.add_request(req) + + with patch.object( + scheduler.connector, + "get_num_new_matched_tokens", + side_effect=[(req.num_tokens, True) for req in reqs], + ): + scheduler.schedule() + + assert all(req.status == RequestStatus.WAITING_FOR_REMOTE_KVS for req in reqs) + scheduler.update_from_output( + scheduler.schedule(), + create_model_runner_output( + [], finished_recving={req.request_id for req in reqs} + ), + ) + output = scheduler.schedule() + for req in reqs: + assert req.status == RequestStatus.RUNNING + assert output.num_scheduled_tokens[req.request_id] > 0 + + def test_deferred_lookup_does_not_block_parked_load(): """A request with a deferred connector lookup must not strand an async load queued behind it. diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index d738e18395..28bfda419a 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -177,6 +177,7 @@ def create_scheduler( vllm_config: VllmConfig, num_blocks: int = 10000, kv_cache_config: KVCacheConfig | None = None, + hash_block_size: int | None = None, ) -> Scheduler | AsyncScheduler: """Initialize Scheduler For Testing.""" block_size = vllm_config.cache_config.block_size @@ -207,6 +208,7 @@ def create_scheduler( log_stats=True, structured_output_manager=StructuredOutputManager(vllm_config), block_size=block_size, + hash_block_size=hash_block_size, ) diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 0758c61225..2603fa10a7 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -344,6 +344,17 @@ class Scheduler(SchedulerInterface): self.cache_config.enable_mamba_shared_prefix_checkpoint ), ) + # Mamba managers already account for their speculative state blocks. + num_spec_decode_tokens = ( + 1 + self.num_spec_tokens + self.num_lookahead_tokens + if self.num_spec_tokens + else 0 + ) + self._spec_decode_step_blocks = sum( + cdiv(num_spec_decode_tokens, manager.block_size) + for manager in self.kv_cache_manager.coordinator.single_type_managers + if not isinstance(manager.kv_cache_spec, MambaSpec) + ) # Bind after construction so connectors can access the cache manager. if self.connector is not None: self.connector.bind_kv_cache_manager(self.kv_cache_manager) @@ -1236,7 +1247,7 @@ class Scheduler(SchedulerInterface): # predictable preemptions. reserved_blocks = ( self._inflight_prefill_reserved_blocks() - + self._spec_decode_step_blocks() + + self._spec_decode_step_blocks ) # Replayed tokens are already counted in the adopted hit; a @@ -3059,9 +3070,12 @@ class Scheduler(SchedulerInterface): def _request_remaining_blocks(self, request: Request) -> int: """Blocks `request` still needs to hold its full sequence and be promoted.""" full_num_tokens = min(request.num_tokens, self.max_model_len) - num_blocks = self.kv_cache_manager.coordinator.get_num_blocks_to_allocate( + num_tokens = full_num_tokens + if self.num_spec_tokens: + num_tokens += 1 + self.num_spec_tokens + self.num_lookahead_tokens + return self.kv_cache_manager.coordinator.get_num_blocks_to_allocate( request_id=request.request_id, - num_tokens=full_num_tokens, + num_tokens=num_tokens, new_computed_blocks=self.kv_cache_manager.empty_kv_cache_blocks.blocks, num_encoder_tokens=0, total_computed_tokens=request.num_computed_tokens, @@ -3070,20 +3084,6 @@ class Scheduler(SchedulerInterface): apply_admission_cap=True, prefill_end=max(request.num_prompt_tokens, request.num_tokens - 1), ) - return num_blocks + self._spec_decode_step_blocks() - - def _spec_decode_step_blocks(self) -> int: - """Number of blocks for the extra KV slots a spec decode step needs. - - When using async kv load, scheduler must reserve enough blocks for - full sequence + the spec decode step, otherwise request cannot be - able to run after the async_load if we are out of kv blocks. - """ - if not self.num_spec_tokens: - return 0 - return cdiv( - 1 + self.num_spec_tokens + self.num_lookahead_tokens, self.block_size - ) def _set_kv_fetch_stage(self, request: Request, stage: str | None) -> None: kv_fetch_stages = self._kv_fetch_stages