[BugFix] Fix P/D preemption race condition (#50297)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Hill
2026-07-30 01:11:18 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 0a31372e5f
commit 437e0b7f8e
7 changed files with 117 additions and 5 deletions
@@ -181,6 +181,18 @@ class KVConnectorBase_V1(ABC):
"""
return False
@property
def requires_kv_delivery(self) -> bool:
"""Whether this connector hands off KV that must be reliably delivered.
If True, a request preempted while its hand-off is still pending is
recomputed rather than allowed to finish and hand off blocks that the
preemption already freed. Defaults to the producer role, since only a
producer hands KV off when a request completes. Best-effort caches
return False, as a dropped save is just a future cache miss.
"""
return self._kv_transfer_config.is_kv_producer
def __init__(
self,
vllm_config: "VllmConfig",
@@ -209,6 +209,10 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
return False
return all(c.prefer_cross_layer_blocks for c in self._connectors)
@property
def requires_kv_delivery(self) -> bool:
return any(c.requires_kv_delivery for c in self._connectors)
@classmethod
def _get_connector_classes_and_configs(
cls, vllm_config: "VllmConfig"
@@ -51,6 +51,12 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
def prefer_cross_layer_blocks(self) -> bool:
return True
@property
def requires_kv_delivery(self) -> bool:
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
# future cache miss, so opt out of the producer-role default.
return False
def __init__(
self,
vllm_config: VllmConfig,
@@ -45,6 +45,12 @@ DEFAULT_CPU_CAPACITY_BYTES = 8 * (1024**3)
class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
"""CPU KV cache offloading with custom kernel transfers and BlockPool LRU."""
@property
def requires_kv_delivery(self) -> bool:
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
# future cache miss, so opt out of the producer-role default.
return False
def __init__(
self,
vllm_config: VllmConfig,
+12 -2
View File
@@ -129,6 +129,9 @@ class Scheduler(SchedulerInterface):
self.connector_prefix_cache_stats: PrefixCacheStats | None = None
self.recompute_kv_load_failures = True
self.defer_block_free = False
# Whether a preempted request's in-flight output must be dropped; see
# KVConnectorBase_V1.requires_kv_delivery.
self.requires_kv_delivery = False
kv_transfer_config = self.vllm_config.kv_transfer_config
if kv_transfer_config is not None:
assert not self.is_encoder_decoder, (
@@ -152,6 +155,8 @@ class Scheduler(SchedulerInterface):
if multiple_inflight_batches and kv_transfer_config.is_kv_consumer:
self.defer_block_free = True
self.requires_kv_delivery = self.connector.requires_kv_delivery
self.kv_event_publisher = EventPublisherFactory.create(
self.kv_events_config,
self.parallel_config.data_parallel_index,
@@ -602,7 +607,11 @@ class Scheduler(SchedulerInterface):
else:
preempted_req = self.running.pop()
self._preempt_request(preempted_req, scheduled_timestamp)
self._preempt_request(
preempted_req,
scheduled_timestamp,
drop_stale_output=self.requires_kv_delivery,
)
preempted_reqs.append(preempted_req)
if preempted_req == request:
# No more request to preempt. Cannot schedule this request.
@@ -1265,7 +1274,8 @@ class Scheduler(SchedulerInterface):
drop_stale_output: drop (rather than deliver) any in-flight output; used
by reset_prefix_cache, whose same-step resume would otherwise deliver
tokens out of order.
tokens out of order, and for connectors with a pending KV hand-off,
which the preemption's block free would leave without valid KV.
"""
assert request.status == RequestStatus.RUNNING, (
"Only running requests can be preempted"