[Bugfix][KV Connector] Fix speculative block reservations across KV cache groups (#59880)
Signed-off-by: bzsuni <bingzhe.sun@daocloud.io>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
from dataclasses import replace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
@@ -761,35 +762,54 @@ def test_async_loads_both_admitted_when_pool_fits():
|
||||
assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
|
||||
|
||||
def test_async_load_reserves_blocks_for_promotion_margin():
|
||||
"""An async load is not admitted unless the blocks its own promotion will
|
||||
need are still free.
|
||||
@pytest.mark.parametrize(
|
||||
("swa_enabled", "block_size", "num_spec_tokens", "num_blocks"),
|
||||
[(False, 16, 3, 8), (True, 16, 3, 17), (True, 64, 31, 40)],
|
||||
ids=["single-group", "multiple-groups", "different-block-sizes"],
|
||||
)
|
||||
def test_async_load_reserves_blocks_for_promotion_margin(
|
||||
swa_enabled, block_size, num_spec_tokens, num_blocks
|
||||
):
|
||||
"""Async loads must leave enough blocks for promotion in every cache group.
|
||||
|
||||
A parked load is allocated without lookahead slots, but promotion pads it
|
||||
to ``1 + num_spec_tokens`` and asks for the lookahead margin on top. Without
|
||||
reserving that margin, two loads can be admitted that together consume the
|
||||
whole pool; the head then fails ``allocate_slots`` forever while the load
|
||||
behind it is never reached, and with nothing running no block is ever freed.
|
||||
|
||||
req_a (4 blocks) and req_b (3 blocks) exactly fill the 7 usable blocks, so
|
||||
req_b must be held back in WAITING holding no blocks, leaving req_a room to
|
||||
be promoted and run.
|
||||
Otherwise two loads can fill the pool and prevent the head from being
|
||||
promoted, with nothing running to free blocks and break the deadlock.
|
||||
"""
|
||||
vllm_config = create_vllm_config(num_speculative_tokens=3)
|
||||
BLOCK_SIZE = vllm_config.cache_config.block_size
|
||||
scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=block_size,
|
||||
num_speculative_tokens=num_spec_tokens,
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
)
|
||||
hash_block_size = 16
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
hash_block_size, swa_enabled=swa_enabled, sw_size=4096, num_blocks=num_blocks
|
||||
)
|
||||
full_group = kv_cache_config.kv_cache_groups[0]
|
||||
full_group.kv_cache_spec = replace(full_group.kv_cache_spec, block_size=block_size)
|
||||
if swa_enabled:
|
||||
sliding_group = kv_cache_config.kv_cache_groups[-1]
|
||||
sliding_group.kv_cache_spec = replace(
|
||||
sliding_group.kv_cache_spec,
|
||||
page_size_padded=full_group.kv_cache_spec.page_size_bytes,
|
||||
)
|
||||
scheduler = create_scheduler(
|
||||
vllm_config,
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_config=kv_cache_config,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req_a = create_request(
|
||||
request_id=1,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=BLOCK_SIZE * 4,
|
||||
block_size=hash_block_size,
|
||||
num_tokens=block_size * 4,
|
||||
do_remote_prefill=True,
|
||||
max_tokens=1,
|
||||
)
|
||||
req_b = create_request(
|
||||
request_id=2,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_tokens=BLOCK_SIZE * 3,
|
||||
block_size=hash_block_size,
|
||||
num_tokens=block_size * 3,
|
||||
do_remote_prefill=True,
|
||||
max_tokens=1,
|
||||
)
|
||||
@@ -800,16 +820,13 @@ def test_async_load_reserves_blocks_for_promotion_margin():
|
||||
with patch.object(
|
||||
scheduler.connector,
|
||||
"get_num_new_matched_tokens",
|
||||
side_effect=[(BLOCK_SIZE * 4, True), (BLOCK_SIZE * 3, True)],
|
||||
side_effect=[(block_size * 4, True), (block_size * 3, True)],
|
||||
):
|
||||
scheduler.schedule()
|
||||
|
||||
assert req_a.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
assert req_b.status == RequestStatus.WAITING
|
||||
req_to_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
|
||||
0
|
||||
].req_to_blocks
|
||||
assert req_b.request_id not in req_to_blocks
|
||||
assert not any(scheduler.kv_cache_manager.get_block_ids(req_b.request_id))
|
||||
|
||||
# req_a's load lands: it must be promotable and actually get scheduled.
|
||||
scheduler.update_from_output(
|
||||
@@ -821,6 +838,74 @@ def test_async_load_reserves_blocks_for_promotion_margin():
|
||||
assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mamba_cache_mode", "mamba_block_size", "num_blocks"),
|
||||
[("none", 128, 18), ("align", 16, 18)],
|
||||
)
|
||||
def test_async_load_does_not_double_reserve_mamba_spec_blocks(
|
||||
mamba_cache_mode, mamba_block_size, num_blocks
|
||||
):
|
||||
"""Two hybrid loads fit when Mamba's speculative states are counted once."""
|
||||
block_size = 16
|
||||
num_spec_tokens = 3
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=mamba_block_size,
|
||||
max_model_len=128,
|
||||
num_speculative_tokens=num_spec_tokens,
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size,
|
||||
mamba_enabled=True,
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
num_blocks=num_blocks,
|
||||
)
|
||||
mamba_group = kv_cache_config.kv_cache_groups[-1]
|
||||
mamba_group.kv_cache_spec = replace(
|
||||
mamba_group.kv_cache_spec,
|
||||
block_size=mamba_block_size,
|
||||
num_speculative_blocks=num_spec_tokens,
|
||||
)
|
||||
scheduler = create_scheduler(
|
||||
vllm_config,
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_config=kv_cache_config,
|
||||
hash_block_size=block_size,
|
||||
)
|
||||
reqs = [
|
||||
create_request(
|
||||
request_id=i,
|
||||
block_size=block_size,
|
||||
num_tokens=num_tokens,
|
||||
do_remote_prefill=True,
|
||||
max_tokens=1,
|
||||
)
|
||||
for i, num_tokens in enumerate((64, 48), 1)
|
||||
]
|
||||
for req in reqs:
|
||||
scheduler.add_request(req)
|
||||
|
||||
with patch.object(
|
||||
scheduler.connector,
|
||||
"get_num_new_matched_tokens",
|
||||
side_effect=[(req.num_tokens, True) for req in reqs],
|
||||
):
|
||||
scheduler.schedule()
|
||||
|
||||
assert all(req.status == RequestStatus.WAITING_FOR_REMOTE_KVS for req in reqs)
|
||||
scheduler.update_from_output(
|
||||
scheduler.schedule(),
|
||||
create_model_runner_output(
|
||||
[], finished_recving={req.request_id for req in reqs}
|
||||
),
|
||||
)
|
||||
output = scheduler.schedule()
|
||||
for req in reqs:
|
||||
assert req.status == RequestStatus.RUNNING
|
||||
assert output.num_scheduled_tokens[req.request_id] > 0
|
||||
|
||||
|
||||
def test_deferred_lookup_does_not_block_parked_load():
|
||||
"""A request with a deferred connector lookup must not strand an async
|
||||
load queued behind it.
|
||||
|
||||
@@ -177,6 +177,7 @@ def create_scheduler(
|
||||
vllm_config: VllmConfig,
|
||||
num_blocks: int = 10000,
|
||||
kv_cache_config: KVCacheConfig | None = None,
|
||||
hash_block_size: int | None = None,
|
||||
) -> Scheduler | AsyncScheduler:
|
||||
"""Initialize Scheduler For Testing."""
|
||||
block_size = vllm_config.cache_config.block_size
|
||||
@@ -207,6 +208,7 @@ def create_scheduler(
|
||||
log_stats=True,
|
||||
structured_output_manager=StructuredOutputManager(vllm_config),
|
||||
block_size=block_size,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -344,6 +344,17 @@ class Scheduler(SchedulerInterface):
|
||||
self.cache_config.enable_mamba_shared_prefix_checkpoint
|
||||
),
|
||||
)
|
||||
# Mamba managers already account for their speculative state blocks.
|
||||
num_spec_decode_tokens = (
|
||||
1 + self.num_spec_tokens + self.num_lookahead_tokens
|
||||
if self.num_spec_tokens
|
||||
else 0
|
||||
)
|
||||
self._spec_decode_step_blocks = sum(
|
||||
cdiv(num_spec_decode_tokens, manager.block_size)
|
||||
for manager in self.kv_cache_manager.coordinator.single_type_managers
|
||||
if not isinstance(manager.kv_cache_spec, MambaSpec)
|
||||
)
|
||||
# Bind after construction so connectors can access the cache manager.
|
||||
if self.connector is not None:
|
||||
self.connector.bind_kv_cache_manager(self.kv_cache_manager)
|
||||
@@ -1236,7 +1247,7 @@ class Scheduler(SchedulerInterface):
|
||||
# predictable preemptions.
|
||||
reserved_blocks = (
|
||||
self._inflight_prefill_reserved_blocks()
|
||||
+ self._spec_decode_step_blocks()
|
||||
+ self._spec_decode_step_blocks
|
||||
)
|
||||
|
||||
# Replayed tokens are already counted in the adopted hit; a
|
||||
@@ -3059,9 +3070,12 @@ class Scheduler(SchedulerInterface):
|
||||
def _request_remaining_blocks(self, request: Request) -> int:
|
||||
"""Blocks `request` still needs to hold its full sequence and be promoted."""
|
||||
full_num_tokens = min(request.num_tokens, self.max_model_len)
|
||||
num_blocks = self.kv_cache_manager.coordinator.get_num_blocks_to_allocate(
|
||||
num_tokens = full_num_tokens
|
||||
if self.num_spec_tokens:
|
||||
num_tokens += 1 + self.num_spec_tokens + self.num_lookahead_tokens
|
||||
return self.kv_cache_manager.coordinator.get_num_blocks_to_allocate(
|
||||
request_id=request.request_id,
|
||||
num_tokens=full_num_tokens,
|
||||
num_tokens=num_tokens,
|
||||
new_computed_blocks=self.kv_cache_manager.empty_kv_cache_blocks.blocks,
|
||||
num_encoder_tokens=0,
|
||||
total_computed_tokens=request.num_computed_tokens,
|
||||
@@ -3070,20 +3084,6 @@ class Scheduler(SchedulerInterface):
|
||||
apply_admission_cap=True,
|
||||
prefill_end=max(request.num_prompt_tokens, request.num_tokens - 1),
|
||||
)
|
||||
return num_blocks + self._spec_decode_step_blocks()
|
||||
|
||||
def _spec_decode_step_blocks(self) -> int:
|
||||
"""Number of blocks for the extra KV slots a spec decode step needs.
|
||||
|
||||
When using async kv load, scheduler must reserve enough blocks for
|
||||
full sequence + the spec decode step, otherwise request cannot be
|
||||
able to run after the async_load if we are out of kv blocks.
|
||||
"""
|
||||
if not self.num_spec_tokens:
|
||||
return 0
|
||||
return cdiv(
|
||||
1 + self.num_spec_tokens + self.num_lookahead_tokens, self.block_size
|
||||
)
|
||||
|
||||
def _set_kv_fetch_stage(self, request: Request, stage: str | None) -> None:
|
||||
kv_fetch_stages = self._kv_fetch_stages
|
||||
|
||||
Reference in New Issue
Block a user