[Bugfix][KV Connector] Fix speculative block reservations across KV cache groups (#59880)

Signed-off-by: bzsuni <bingzhe.sun@daocloud.io>
This commit is contained in:
bzsuni
2026-10-10 20:55:41 -07:00
committed by GitHub
parent b89852aad0
commit 01abe975b5
3 changed files with 128 additions and 41 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from dataclasses import replace
from unittest.mock import patch
import pytest
@@ -761,35 +762,54 @@ def test_async_loads_both_admitted_when_pool_fits():
assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
def test_async_load_reserves_blocks_for_promotion_margin():
"""An async load is not admitted unless the blocks its own promotion will
need are still free.
@pytest.mark.parametrize(
("swa_enabled", "block_size", "num_spec_tokens", "num_blocks"),
[(False, 16, 3, 8), (True, 16, 3, 17), (True, 64, 31, 40)],
ids=["single-group", "multiple-groups", "different-block-sizes"],
)
def test_async_load_reserves_blocks_for_promotion_margin(
swa_enabled, block_size, num_spec_tokens, num_blocks
):
"""Async loads must leave enough blocks for promotion in every cache group.
A parked load is allocated without lookahead slots, but promotion pads it
to ``1 + num_spec_tokens`` and asks for the lookahead margin on top. Without
reserving that margin, two loads can be admitted that together consume the
whole pool; the head then fails ``allocate_slots`` forever while the load
behind it is never reached, and with nothing running no block is ever freed.
req_a (4 blocks) and req_b (3 blocks) exactly fill the 7 usable blocks, so
req_b must be held back in WAITING holding no blocks, leaving req_a room to
be promoted and run.
Otherwise two loads can fill the pool and prevent the head from being
promoted, with nothing running to free blocks and break the deadlock.
"""
vllm_config = create_vllm_config(num_speculative_tokens=3)
BLOCK_SIZE = vllm_config.cache_config.block_size
scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7
vllm_config = create_vllm_config(
block_size=block_size,
num_speculative_tokens=num_spec_tokens,
disable_hybrid_kv_cache_manager=False,
)
hash_block_size = 16
kv_cache_config = make_kv_cache_config(
hash_block_size, swa_enabled=swa_enabled, sw_size=4096, num_blocks=num_blocks
)
full_group = kv_cache_config.kv_cache_groups[0]
full_group.kv_cache_spec = replace(full_group.kv_cache_spec, block_size=block_size)
if swa_enabled:
sliding_group = kv_cache_config.kv_cache_groups[-1]
sliding_group.kv_cache_spec = replace(
sliding_group.kv_cache_spec,
page_size_padded=full_group.kv_cache_spec.page_size_bytes,
)
scheduler = create_scheduler(
vllm_config,
num_blocks=num_blocks,
kv_cache_config=kv_cache_config,
hash_block_size=hash_block_size,
)
req_a = create_request(
request_id=1,
block_size=BLOCK_SIZE,
num_tokens=BLOCK_SIZE * 4,
block_size=hash_block_size,
num_tokens=block_size * 4,
do_remote_prefill=True,
max_tokens=1,
)
req_b = create_request(
request_id=2,
block_size=BLOCK_SIZE,
num_tokens=BLOCK_SIZE * 3,
block_size=hash_block_size,
num_tokens=block_size * 3,
do_remote_prefill=True,
max_tokens=1,
)
@@ -800,16 +820,13 @@ def test_async_load_reserves_blocks_for_promotion_margin():
with patch.object(
scheduler.connector,
"get_num_new_matched_tokens",
side_effect=[(BLOCK_SIZE * 4, True), (BLOCK_SIZE * 3, True)],
side_effect=[(block_size * 4, True), (block_size * 3, True)],
):
scheduler.schedule()
assert req_a.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert req_b.status == RequestStatus.WAITING
req_to_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
0
].req_to_blocks
assert req_b.request_id not in req_to_blocks
assert not any(scheduler.kv_cache_manager.get_block_ids(req_b.request_id))
# req_a's load lands: it must be promotable and actually get scheduled.
scheduler.update_from_output(
@@ -821,6 +838,74 @@ def test_async_load_reserves_blocks_for_promotion_margin():
assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0
@pytest.mark.parametrize(
("mamba_cache_mode", "mamba_block_size", "num_blocks"),
[("none", 128, 18), ("align", 16, 18)],
)
def test_async_load_does_not_double_reserve_mamba_spec_blocks(
mamba_cache_mode, mamba_block_size, num_blocks
):
"""Two hybrid loads fit when Mamba's speculative states are counted once."""
block_size = 16
num_spec_tokens = 3
vllm_config = create_vllm_config(
block_size=mamba_block_size,
max_model_len=128,
num_speculative_tokens=num_spec_tokens,
disable_hybrid_kv_cache_manager=False,
)
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
kv_cache_config = make_kv_cache_config(
block_size,
mamba_enabled=True,
mamba_cache_mode=mamba_cache_mode,
num_blocks=num_blocks,
)
mamba_group = kv_cache_config.kv_cache_groups[-1]
mamba_group.kv_cache_spec = replace(
mamba_group.kv_cache_spec,
block_size=mamba_block_size,
num_speculative_blocks=num_spec_tokens,
)
scheduler = create_scheduler(
vllm_config,
num_blocks=num_blocks,
kv_cache_config=kv_cache_config,
hash_block_size=block_size,
)
reqs = [
create_request(
request_id=i,
block_size=block_size,
num_tokens=num_tokens,
do_remote_prefill=True,
max_tokens=1,
)
for i, num_tokens in enumerate((64, 48), 1)
]
for req in reqs:
scheduler.add_request(req)
with patch.object(
scheduler.connector,
"get_num_new_matched_tokens",
side_effect=[(req.num_tokens, True) for req in reqs],
):
scheduler.schedule()
assert all(req.status == RequestStatus.WAITING_FOR_REMOTE_KVS for req in reqs)
scheduler.update_from_output(
scheduler.schedule(),
create_model_runner_output(
[], finished_recving={req.request_id for req in reqs}
),
)
output = scheduler.schedule()
for req in reqs:
assert req.status == RequestStatus.RUNNING
assert output.num_scheduled_tokens[req.request_id] > 0
def test_deferred_lookup_does_not_block_parked_load():
"""A request with a deferred connector lookup must not strand an async
load queued behind it.
+2
View File
@@ -177,6 +177,7 @@ def create_scheduler(
vllm_config: VllmConfig,
num_blocks: int = 10000,
kv_cache_config: KVCacheConfig | None = None,
hash_block_size: int | None = None,
) -> Scheduler | AsyncScheduler:
"""Initialize Scheduler For Testing."""
block_size = vllm_config.cache_config.block_size
@@ -207,6 +208,7 @@ def create_scheduler(
log_stats=True,
structured_output_manager=StructuredOutputManager(vllm_config),
block_size=block_size,
hash_block_size=hash_block_size,
)
+17 -17
View File
@@ -344,6 +344,17 @@ class Scheduler(SchedulerInterface):
self.cache_config.enable_mamba_shared_prefix_checkpoint
),
)
# Mamba managers already account for their speculative state blocks.
num_spec_decode_tokens = (
1 + self.num_spec_tokens + self.num_lookahead_tokens
if self.num_spec_tokens
else 0
)
self._spec_decode_step_blocks = sum(
cdiv(num_spec_decode_tokens, manager.block_size)
for manager in self.kv_cache_manager.coordinator.single_type_managers
if not isinstance(manager.kv_cache_spec, MambaSpec)
)
# Bind after construction so connectors can access the cache manager.
if self.connector is not None:
self.connector.bind_kv_cache_manager(self.kv_cache_manager)
@@ -1236,7 +1247,7 @@ class Scheduler(SchedulerInterface):
# predictable preemptions.
reserved_blocks = (
self._inflight_prefill_reserved_blocks()
+ self._spec_decode_step_blocks()
+ self._spec_decode_step_blocks
)
# Replayed tokens are already counted in the adopted hit; a
@@ -3059,9 +3070,12 @@ class Scheduler(SchedulerInterface):
def _request_remaining_blocks(self, request: Request) -> int:
"""Blocks `request` still needs to hold its full sequence and be promoted."""
full_num_tokens = min(request.num_tokens, self.max_model_len)
num_blocks = self.kv_cache_manager.coordinator.get_num_blocks_to_allocate(
num_tokens = full_num_tokens
if self.num_spec_tokens:
num_tokens += 1 + self.num_spec_tokens + self.num_lookahead_tokens
return self.kv_cache_manager.coordinator.get_num_blocks_to_allocate(
request_id=request.request_id,
num_tokens=full_num_tokens,
num_tokens=num_tokens,
new_computed_blocks=self.kv_cache_manager.empty_kv_cache_blocks.blocks,
num_encoder_tokens=0,
total_computed_tokens=request.num_computed_tokens,
@@ -3070,20 +3084,6 @@ class Scheduler(SchedulerInterface):
apply_admission_cap=True,
prefill_end=max(request.num_prompt_tokens, request.num_tokens - 1),
)
return num_blocks + self._spec_decode_step_blocks()
def _spec_decode_step_blocks(self) -> int:
"""Number of blocks for the extra KV slots a spec decode step needs.
When using async kv load, scheduler must reserve enough blocks for
full sequence + the spec decode step, otherwise request cannot be
able to run after the async_load if we are out of kv blocks.
"""
if not self.num_spec_tokens:
return 0
return cdiv(
1 + self.num_spec_tokens + self.num_lookahead_tokens, self.block_size
)
def _set_kv_fetch_stage(self, request: Request, stage: str | None) -> None:
kv_fetch_stages = self._kv_fetch_stages