[BugFix][KV Connector] Fix Deadlock with KVConnector + MTP under KV Pressure (#57104)

Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Robert Shaw
2026-09-16 22:22:07 +00:00
committed by GitHub
co-authored by Claude
parent 5a6ccc5892
commit 6c3d9816af
3 changed files with 92 additions and 5 deletions
+24 -5
View File
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
from vllm.multimodal.encoder_budget import MultiModalBudget
from vllm.multimodal.utils import get_mm_features_in_window
from vllm.utils.math_utils import cdiv
from vllm.v1.core.encoder_cache_manager import (
EncoderCacheManager,
EncoderDecoderCacheManager,
@@ -1140,9 +1141,13 @@ class Scheduler(SchedulerInterface):
if load_kv_async:
# An async load holds its blocks for the whole transfer with
# no forward progress and isn't preemptible here. Admit it
# only if it fits in (free - other in-flight reservations), to
# avoid deadlock and predictable preemptions.
reserved_blocks = self._inflight_prefill_reserved_blocks()
# only if it fits in (free - other in-flight reservations)
# plus its own spec decode step blocks, to avoid deadlock and
# predictable preemptions.
reserved_blocks = (
self._inflight_prefill_reserved_blocks()
+ self._spec_decode_step_blocks()
)
new_blocks = self.kv_cache_manager.allocate_slots(
request,
@@ -2892,9 +2897,9 @@ class Scheduler(SchedulerInterface):
return delay_free or partial_tail_delay, kv_xfer_params
def _request_remaining_blocks(self, request: Request) -> int:
"""Blocks `request` still needs to allocate to hold its full sequence."""
"""Blocks `request` still needs to hold its full sequence and be promoted."""
full_num_tokens = min(request.num_tokens, self.max_model_len)
return self.kv_cache_manager.coordinator.get_num_blocks_to_allocate(
num_blocks = self.kv_cache_manager.coordinator.get_num_blocks_to_allocate(
request_id=request.request_id,
num_tokens=full_num_tokens,
new_computed_blocks=self.kv_cache_manager.empty_kv_cache_blocks.blocks,
@@ -2904,6 +2909,20 @@ class Scheduler(SchedulerInterface):
num_tokens_main_model=full_num_tokens,
apply_admission_cap=True,
)
return num_blocks + self._spec_decode_step_blocks()
def _spec_decode_step_blocks(self) -> int:
"""Number of blocks for the extra KV slots a spec decode step needs.
When using async kv load, scheduler must reserve enough blocks for
full sequence + the spec decode step, otherwise request cannot be
able to run after the async_load if we are out of kv blocks.
"""
if not self.num_spec_tokens:
return 0
return cdiv(
1 + self.num_spec_tokens + self.num_lookahead_tokens, self.block_size
)
def _inflight_prefill_reserved_blocks(self) -> int:
"""Num blocks in-flight prefills still need to finish (their reservation)."""