[BugFix] Resolve multiple async kv load deadlock (#44560)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-06 23:05:47 +00:00
committed by GitHub
parent 062b05ff3a
commit 3b3d5287fa
6 changed files with 214 additions and 1 deletions
+78
View File
@@ -4474,3 +4474,81 @@ def test_ec_connector_pending_prefetch_only_checks_future_mm_features():
f"Expected only {HASH_FUTURE!r} from future mm feature filtering, "
f"got {future_hashes!r}. Past/boundary features must be filtered out."
)
def test_async_load_reservation_prevents_wedge_e2e():
"""Same wedge scenario as PR #40968's lateral-preemption e2e test, but
resolved by reservation-based admission control instead of preemption.
A (8 blocks) and B (5 blocks) both want an async KV load, sharing a 4-block
prefix, in a 10-block pool (9 usable). Admitting both loads would wedge:
once their recvs finish neither can complete its local prefill (8+5 > 9).
Here the reservation gate refuses to admit B's load while A's full sequence
is still reserved, so B never holds blocks and A is free to complete - no
deadlock, and (unlike lateral preemption) B is never preempted.
"""
BLOCK_SIZE = 16
A_TOKENS = BLOCK_SIZE * 8 # bigger request
B_TOKENS = BLOCK_SIZE * 5 # smaller request
MATCHED_TOKENS = BLOCK_SIZE * 4 # 4-block prefix loaded for both
NUM_BLOCKS = 10 # 9 usable; both prefixes fit, but not both full sequences
scheduler = create_scheduler(
block_size=BLOCK_SIZE,
num_blocks=NUM_BLOCKS,
max_num_seqs=4,
max_num_batched_tokens=A_TOKENS * 2,
use_kv_connector=mock_kv(matched_tokens=MATCHED_TOKENS, is_async=True),
)
[a] = create_requests(
num_requests=1, num_tokens=A_TOKENS, block_size=BLOCK_SIZE, req_ids=["a"]
)
[b] = create_requests(
num_requests=1, num_tokens=B_TOKENS, block_size=BLOCK_SIZE, req_ids=["b"]
)
scheduler.add_request(a)
scheduler.add_request(b)
EMPTY_OUTPUT = ModelRunnerOutput(
req_ids=[],
req_id_to_index={},
sampled_token_ids=[],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
)
req_to_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
0
].req_to_blocks
# Step 1: A's load is admitted; B's is held back by the reservation (B never
# holds blocks, so the wedge precondition - both holding prefixes - is gone).
out1 = scheduler.schedule()
assert a.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert a.num_computed_tokens == MATCHED_TOKENS
assert b.status == RequestStatus.WAITING
assert b.request_id not in req_to_blocks
assert len(scheduler.running) == 0
scheduler.update_from_output(out1, EMPTY_OUTPUT)
# Step 2: nothing changes until A's recv lands.
out2 = scheduler.schedule()
assert len(scheduler.running) == 0
a_finished = dataclasses.replace(
EMPTY_OUTPUT,
kv_connector_output=KVConnectorOutput(finished_recving=[a.request_id]),
)
scheduler.update_from_output(out2, a_finished)
# Step 3: A makes forward progress straight to RUNNING - no preemption was
# needed because B never wedged it.
out3 = scheduler.schedule()
assert a.status == RequestStatus.RUNNING
assert a in scheduler.running
assert a.request_id in {req.req_id for req in out3.scheduled_new_reqs}
assert b.status == RequestStatus.WAITING
assert b.num_preemptions == 0
assert b.request_id not in req_to_blocks
@@ -180,6 +180,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable):
delay_cache_blocks: bool = False,
num_encoder_tokens: int = 0,
full_sequence_must_fit: bool = False,
reserved_blocks: int = 0,
):
ret = original_allocate_slots_fn(
self,
@@ -192,6 +193,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable):
delay_cache_blocks,
num_encoder_tokens,
full_sequence_must_fit,
reserved_blocks,
)
if cur_step_action is not None:
cur_block_ids = self.coordinator.single_type_managers[0].req_to_blocks[
@@ -655,3 +655,82 @@ def test_p_side_chunked_prefill_mamba(mock_platform):
outputs = engine_core_outputs[0].outputs
assert len(outputs) == 1
assert outputs[0].finish_reason == FinishReason.LENGTH
def test_async_load_reserves_blocks_for_inflight():
"""A second async KV-connector load is not admitted if its initial
allocation would consume blocks reserved for an already in-flight sequence.
req_a gets a 1-block prefix (full sequence = 4 blocks), reserving 3 more.
req_b would need a 4-block initial allocation, but only
(free - req_a's 3-block reservation) = 3 blocks are available to it, so it is
held back in WAITING (holding no blocks) rather than wedging req_a.
"""
vllm_config = create_vllm_config()
BLOCK_SIZE = vllm_config.cache_config.block_size
scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7
req_a = create_request(
request_id=1,
block_size=BLOCK_SIZE,
num_tokens=BLOCK_SIZE * 4,
do_remote_prefill=True,
num_remote_blocks=1,
)
req_b = create_request(
request_id=2,
block_size=BLOCK_SIZE,
num_tokens=BLOCK_SIZE * 5,
do_remote_prefill=True,
num_remote_blocks=1,
)
scheduler.add_request(req_a)
scheduler.add_request(req_b)
# Partial external matches: req_a loads 1 block, req_b loads 4 blocks.
with patch.object(
scheduler.connector,
"get_num_new_matched_tokens",
side_effect=[(BLOCK_SIZE, True), (BLOCK_SIZE * 4, True)],
):
scheduler.schedule()
assert req_a.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert req_b.status == RequestStatus.WAITING
req_to_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
0
].req_to_blocks
assert req_a.request_id in req_to_blocks
assert req_b.request_id not in req_to_blocks
def test_async_loads_both_admitted_when_pool_fits():
"""Sanity: with a pool large enough, the reservation gate admits both async
loads (it is not over-conservative)."""
vllm_config = create_vllm_config()
BLOCK_SIZE = vllm_config.cache_config.block_size
scheduler = create_scheduler(vllm_config, num_blocks=64)
reqs = [
create_request(
request_id=i,
block_size=BLOCK_SIZE,
num_tokens=BLOCK_SIZE * 5,
do_remote_prefill=True,
num_remote_blocks=1,
)
for i in (1, 2)
]
for req in reqs:
scheduler.add_request(req)
with patch.object(
scheduler.connector,
"get_num_new_matched_tokens",
side_effect=[(BLOCK_SIZE, True), (BLOCK_SIZE, True)],
):
scheduler.schedule()
for req in reqs:
assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
+1
View File
@@ -56,6 +56,7 @@ def assert_scheduler_empty(scheduler: Scheduler):
assert len(scheduler.running) == 0
assert len(scheduler.finished_req_ids) == 0
assert len(scheduler.finished_recving_kv_req_ids) == 0
assert len(scheduler._inflight_prefills) == 0
# EncoderCacheManager.
assert len(scheduler.encoder_cache_manager.freed) == 0