[BugFix] Fix P/D preemption race condition (#50297)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
0a31372e5f
commit
437e0b7f8e
@@ -181,6 +181,18 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return False
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
"""Whether this connector hands off KV that must be reliably delivered.
|
||||
|
||||
If True, a request preempted while its hand-off is still pending is
|
||||
recomputed rather than allowed to finish and hand off blocks that the
|
||||
preemption already freed. Defaults to the producer role, since only a
|
||||
producer hands KV off when a request completes. Best-effort caches
|
||||
return False, as a dropped save is just a future cache miss.
|
||||
"""
|
||||
return self._kv_transfer_config.is_kv_producer
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
|
||||
@@ -209,6 +209,10 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return False
|
||||
return all(c.prefer_cross_layer_blocks for c in self._connectors)
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
return any(c.requires_kv_delivery for c in self._connectors)
|
||||
|
||||
@classmethod
|
||||
def _get_connector_classes_and_configs(
|
||||
cls, vllm_config: "VllmConfig"
|
||||
|
||||
@@ -51,6 +51,12 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def prefer_cross_layer_blocks(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
|
||||
# future cache miss, so opt out of the producer-role default.
|
||||
return False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
|
||||
@@ -45,6 +45,12 @@ DEFAULT_CPU_CAPACITY_BYTES = 8 * (1024**3)
|
||||
class SimpleCPUOffloadConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""CPU KV cache offloading with custom kernel transfers and BlockPool LRU."""
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
|
||||
# future cache miss, so opt out of the producer-role default.
|
||||
return False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
|
||||
Reference in New Issue
Block a user