[BugFix] Fix P/D preemption race condition (#50297)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
0a31372e5f
commit
437e0b7f8e
@@ -181,6 +181,18 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return False
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
"""Whether this connector hands off KV that must be reliably delivered.
|
||||
|
||||
If True, a request preempted while its hand-off is still pending is
|
||||
recomputed rather than allowed to finish and hand off blocks that the
|
||||
preemption already freed. Defaults to the producer role, since only a
|
||||
producer hands KV off when a request completes. Best-effort caches
|
||||
return False, as a dropped save is just a future cache miss.
|
||||
"""
|
||||
return self._kv_transfer_config.is_kv_producer
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
|
||||
@@ -209,6 +209,10 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return False
|
||||
return all(c.prefer_cross_layer_blocks for c in self._connectors)
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
return any(c.requires_kv_delivery for c in self._connectors)
|
||||
|
||||
@classmethod
|
||||
def _get_connector_classes_and_configs(
|
||||
cls, vllm_config: "VllmConfig"
|
||||
|
||||
@@ -51,6 +51,12 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def prefer_cross_layer_blocks(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
|
||||
# future cache miss, so opt out of the producer-role default.
|
||||
return False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
|
||||
@@ -45,6 +45,12 @@ DEFAULT_CPU_CAPACITY_BYTES = 8 * (1024**3)
|
||||
class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""CPU KV cache offloading with custom kernel transfers and BlockPool LRU."""
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
|
||||
# future cache miss, so opt out of the producer-role default.
|
||||
return False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
|
||||
@@ -129,6 +129,9 @@ class Scheduler(SchedulerInterface):
|
||||
self.connector_prefix_cache_stats: PrefixCacheStats | None = None
|
||||
self.recompute_kv_load_failures = True
|
||||
self.defer_block_free = False
|
||||
# Whether a preempted request's in-flight output must be dropped; see
|
||||
# KVConnectorBase_V1.requires_kv_delivery.
|
||||
self.requires_kv_delivery = False
|
||||
kv_transfer_config = self.vllm_config.kv_transfer_config
|
||||
if kv_transfer_config is not None:
|
||||
assert not self.is_encoder_decoder, (
|
||||
@@ -152,6 +155,8 @@ class Scheduler(SchedulerInterface):
|
||||
if multiple_inflight_batches and kv_transfer_config.is_kv_consumer:
|
||||
self.defer_block_free = True
|
||||
|
||||
self.requires_kv_delivery = self.connector.requires_kv_delivery
|
||||
|
||||
self.kv_event_publisher = EventPublisherFactory.create(
|
||||
self.kv_events_config,
|
||||
self.parallel_config.data_parallel_index,
|
||||
@@ -602,7 +607,11 @@ class Scheduler(SchedulerInterface):
|
||||
else:
|
||||
preempted_req = self.running.pop()
|
||||
|
||||
self._preempt_request(preempted_req, scheduled_timestamp)
|
||||
self._preempt_request(
|
||||
preempted_req,
|
||||
scheduled_timestamp,
|
||||
drop_stale_output=self.requires_kv_delivery,
|
||||
)
|
||||
preempted_reqs.append(preempted_req)
|
||||
if preempted_req == request:
|
||||
# No more request to preempt. Cannot schedule this request.
|
||||
@@ -1265,7 +1274,8 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
drop_stale_output: drop (rather than deliver) any in-flight output; used
|
||||
by reset_prefix_cache, whose same-step resume would otherwise deliver
|
||||
tokens out of order.
|
||||
tokens out of order, and for connectors with a pending KV hand-off,
|
||||
which the preemption's block free would leave without valid KV.
|
||||
"""
|
||||
assert request.status == RequestStatus.RUNNING, (
|
||||
"Only running requests can be preempted"
|
||||
|
||||
Reference in New Issue
Block a user