[BugFix][KV Connector] Fix Deadlock with KVConnector + MTP under KV Pressure (#57104)

Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Robert Shaw
2026-09-16 22:22:07 +00:00
committed by GitHub
co-authored by Claude
parent 5a6ccc5892
commit 6c3d9816af
3 changed files with 92 additions and 5 deletions
@@ -731,3 +731,63 @@ def test_async_loads_both_admitted_when_pool_fits():
for req in reqs:
assert req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
def test_async_load_reserves_blocks_for_promotion_margin():
"""An async load is not admitted unless the blocks its own promotion will
need are still free.
A parked load is allocated without lookahead slots, but promotion pads it
to ``1 + num_spec_tokens`` and asks for the lookahead margin on top. Without
reserving that margin, two loads can be admitted that together consume the
whole pool; the head then fails ``allocate_slots`` forever while the load
behind it is never reached, and with nothing running no block is ever freed.
req_a (4 blocks) and req_b (3 blocks) exactly fill the 7 usable blocks, so
req_b must be held back in WAITING holding no blocks, leaving req_a room to
be promoted and run.
"""
vllm_config = create_vllm_config(num_speculative_tokens=3)
BLOCK_SIZE = vllm_config.cache_config.block_size
scheduler = create_scheduler(vllm_config, num_blocks=8) # usable = 7
req_a = create_request(
request_id=1,
block_size=BLOCK_SIZE,
num_tokens=BLOCK_SIZE * 4,
do_remote_prefill=True,
max_tokens=1,
)
req_b = create_request(
request_id=2,
block_size=BLOCK_SIZE,
num_tokens=BLOCK_SIZE * 3,
do_remote_prefill=True,
max_tokens=1,
)
scheduler.add_request(req_a)
scheduler.add_request(req_b)
# Both get a full external hit, so each would hold its whole prompt.
with patch.object(
scheduler.connector,
"get_num_new_matched_tokens",
side_effect=[(BLOCK_SIZE * 4, True), (BLOCK_SIZE * 3, True)],
):
scheduler.schedule()
assert req_a.status == RequestStatus.WAITING_FOR_REMOTE_KVS
assert req_b.status == RequestStatus.WAITING
req_to_blocks = scheduler.kv_cache_manager.coordinator.single_type_managers[
0
].req_to_blocks
assert req_b.request_id not in req_to_blocks
# req_a's load lands: it must be promotable and actually get scheduled.
scheduler.update_from_output(
scheduler.schedule(),
create_model_runner_output([], finished_recving={req_a.request_id}),
)
scheduler_output = scheduler.schedule()
assert req_a.status == RequestStatus.RUNNING
assert scheduler_output.num_scheduled_tokens[req_a.request_id] > 0
+8
View File
@@ -17,6 +17,7 @@ from vllm.config import (
KVTransferConfig,
ModelConfig,
SchedulerConfig,
SpeculativeConfig,
VllmConfig,
)
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
@@ -106,6 +107,7 @@ def create_vllm_config(
kv_connector_module_path: str | None = None,
kv_role: str = "kv_consumer",
disable_hybrid_kv_cache_manager: bool | None = None,
num_speculative_tokens: int | None = None,
) -> VllmConfig:
"""Initialize VllmConfig For Testing."""
model_config = ModelConfig(
@@ -141,6 +143,11 @@ def create_vllm_config(
kv_load_failure_policy=kv_load_failure_policy,
)
attention_config = AttentionConfig(backend=attention_backend)
speculative_config = (
SpeculativeConfig(model="ngram", num_speculative_tokens=num_speculative_tokens)
if num_speculative_tokens is not None
else None
)
return VllmConfig(
scheduler_config=scheduler_config,
model_config=model_config,
@@ -148,6 +155,7 @@ def create_vllm_config(
kv_transfer_config=kv_transfer_config,
device_config=DeviceConfig("cpu"),
attention_config=attention_config,
speculative_config=speculative_config,
)