[Bugfix][NIXL] Recover locally invalidated pull peer metadata (#55471)
Signed-off-by: xijiade.aihemaiti <3146335281@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yannik Hinteregger <37209495+YannikHinteregger@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Nick Hill
Yannik Hinteregger
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9bf4e51030
commit
9465e1c783
@@ -101,6 +101,11 @@ logger = init_logger(__name__)
|
||||
|
||||
_SHARED_REGION_GROUP_ID = -1
|
||||
|
||||
# Re-handshake delay bounds for a pull peer whose local metadata keeps getting
|
||||
# invalidated, eg due to a persistent RDMA fault.
|
||||
_INVALID_PEER_MIN_BACKOFF_S = 1.0
|
||||
_INVALID_PEER_MAX_BACKOFF_S = 60.0
|
||||
|
||||
|
||||
def _region_sort_key(layer_name: str) -> tuple[tuple[int, int | str], ...]:
|
||||
"""Sort transfer regions in model-layer order, then by cache name."""
|
||||
@@ -685,6 +690,12 @@ class NixlBaseConnectorWorker:
|
||||
self._remote_agents: dict[EngineId, dict[tuple[int, int], str]] = defaultdict(
|
||||
dict
|
||||
)
|
||||
# Failed reads need a local metadata probe; invalid peers wait for drain.
|
||||
self._failed_remote_engines: set[EngineId] = set()
|
||||
self._invalid_remote_engines: set[EngineId] = set()
|
||||
# Invalid engine -> (backoff, earliest release time). Kept across release
|
||||
# so that repeat invalidations back off.
|
||||
self._invalid_engine_backoff: dict[EngineId, tuple[float, float]] = {}
|
||||
# Map of engine_id -> clock offset.
|
||||
self._engine_clock_offset: dict[EngineId, float] = {}
|
||||
|
||||
@@ -3029,7 +3040,7 @@ class NixlBaseConnectorWorker:
|
||||
# Handle timeout to avoid stranding blocks on remote.
|
||||
self._reap_expired_send_leases(done_sending)
|
||||
|
||||
self._cleanup_replaced_remote_engines()
|
||||
self._cleanup_remote_engines()
|
||||
return KVConnectorTransferResults(
|
||||
finished_sending=done_sending,
|
||||
finished_recving=done_recving,
|
||||
@@ -3195,6 +3206,12 @@ class NixlBaseConnectorWorker:
|
||||
"""
|
||||
if record_failed_transfer:
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
if (
|
||||
self._TRANSFER_MODE == "pull"
|
||||
and (meta := self._recving_metadata.get(req_id)) is not None
|
||||
and meta.remote is not None
|
||||
):
|
||||
self._failed_remote_engines.add(meta.remote.engine_id)
|
||||
if failed_req_ids is not None:
|
||||
failed_req_ids.add(req_id)
|
||||
return handle is None or self._try_release_xfer_handle(req_id, handle)
|
||||
@@ -3642,15 +3659,38 @@ class NixlBaseConnectorWorker:
|
||||
engine_id,
|
||||
)
|
||||
|
||||
def _cleanup_replaced_remote_engines(self) -> None:
|
||||
"""Release replaced pull peers once requests and handshakes have drained."""
|
||||
def _cleanup_remote_engines(self) -> None:
|
||||
"""Release replaced or locally invalid pull peers after their reads drain."""
|
||||
# TODO: Also handle push mode, which handshakes in both directions.
|
||||
if self._TRANSFER_MODE != "pull":
|
||||
return
|
||||
# A background handshake may be loading native metadata with the GIL
|
||||
# released. Defer both probing and cleanup until it has finished.
|
||||
with self._handshake_lock:
|
||||
if self._handshake_futures:
|
||||
return
|
||||
replaced = self._remote_agents.keys() - self._engine_by_address.values()
|
||||
if not replaced:
|
||||
now = time.perf_counter()
|
||||
for engine_id in self._failed_remote_engines - self._invalid_remote_engines:
|
||||
agents = self._remote_agents.get(engine_id, {}).values()
|
||||
try:
|
||||
# No descriptors: query local existence, not peer health.
|
||||
if not all(map(self.nixl_wrapper.check_remote_metadata, agents)):
|
||||
self._mark_remote_engine_invalid(engine_id, now)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Could not check local NIXL metadata for engine %s.",
|
||||
engine_id,
|
||||
exc_info=True,
|
||||
)
|
||||
self._failed_remote_engines.clear()
|
||||
# Invalid peers keep rejecting new reads until their backoff has elapsed.
|
||||
invalid = {
|
||||
engine_id
|
||||
for engine_id in self._invalid_remote_engines
|
||||
if self._invalid_engine_backoff.get(engine_id, (0.0, 0.0))[1] <= now
|
||||
}
|
||||
if not replaced and not invalid:
|
||||
return
|
||||
# _recving_metadata is only accessed from this (main) thread.
|
||||
busy = {
|
||||
@@ -3658,9 +3698,42 @@ class NixlBaseConnectorWorker:
|
||||
for meta in self._recving_metadata.values()
|
||||
if meta.remote is not None
|
||||
}
|
||||
for engine_id in replaced - busy:
|
||||
# Replaced peers can still serve queued requests. Invalid peers reject
|
||||
# new reads, but must retain descriptors until outstanding handles end.
|
||||
invalid -= self._engines_with_inflight_transfers()
|
||||
for engine_id in (replaced - busy) | invalid:
|
||||
self._cleanup_remote_engine(engine_id, log_eviction=False)
|
||||
logger.info("Released NIXL state for replaced remote engine %s.", engine_id)
|
||||
logger.info(
|
||||
"Released NIXL state for replaced or locally invalid remote engine %s.",
|
||||
engine_id,
|
||||
)
|
||||
|
||||
def _mark_remote_engine_invalid(self, engine_id: EngineId, now: float) -> None:
|
||||
"""Schedule release of an engine whose local NIXL metadata is gone.
|
||||
|
||||
The first invalidation is released as soon as reads drain. Repeats
|
||||
before the peer has stayed healthy for the max backoff double the
|
||||
delay, during which reads fail fast instead of re-handshaking.
|
||||
"""
|
||||
# Forget engines that have stayed healthy long enough.
|
||||
for eid, (_, release_at) in list(self._invalid_engine_backoff.items()):
|
||||
if now - release_at > _INVALID_PEER_MAX_BACKOFF_S:
|
||||
del self._invalid_engine_backoff[eid]
|
||||
prev = self._invalid_engine_backoff.get(engine_id)
|
||||
backoff = 0.0
|
||||
if prev is not None:
|
||||
backoff = min(
|
||||
max(2 * prev[0], _INVALID_PEER_MIN_BACKOFF_S),
|
||||
_INVALID_PEER_MAX_BACKOFF_S,
|
||||
)
|
||||
self._invalid_engine_backoff[engine_id] = (backoff, now + backoff)
|
||||
self._invalid_remote_engines.add(engine_id)
|
||||
logger.warning(
|
||||
"Local NIXL metadata for remote engine %s was invalidated; "
|
||||
"re-handshaking after reads drain%s.",
|
||||
engine_id,
|
||||
f" and a {backoff:.0f}s backoff" if backoff else "",
|
||||
)
|
||||
|
||||
def _cleanup_remote_engine(
|
||||
self, engine_id: EngineId, *, log_eviction: bool = True
|
||||
@@ -3668,8 +3741,8 @@ class NixlBaseConnectorWorker:
|
||||
"""Remove all state for a single remote engine.
|
||||
|
||||
Releases NIXL resources (dlist handles, remote agents) and clears
|
||||
all per-engine data structures. Used by TTL eviction, replaced peer
|
||||
cleanup and shutdown.
|
||||
all per-engine data structures. Used by TTL eviction, replaced/invalid
|
||||
peer cleanup and shutdown.
|
||||
"""
|
||||
assert engine_id in self._remote_agents
|
||||
|
||||
@@ -3685,6 +3758,9 @@ class NixlBaseConnectorWorker:
|
||||
for agent_name in agents.values():
|
||||
self.nixl_wrapper.remove_remote_agent(agent_name)
|
||||
|
||||
self._failed_remote_engines.discard(engine_id)
|
||||
self._invalid_remote_engines.discard(engine_id)
|
||||
|
||||
self.kv_caches_base_addr.pop(engine_id, None)
|
||||
self.dst_num_blocks.pop(engine_id, None)
|
||||
self.dst_region_num_blocks.pop(engine_id, None)
|
||||
|
||||
@@ -72,26 +72,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
# Aborted cleanup requests have already been removed from the scheduler.
|
||||
if meta.awaiting_kvs or any(meta.local_block_ids):
|
||||
self._recving_metadata[req_id] = meta
|
||||
if remote_engine_id not in self._remote_agents:
|
||||
# Initiate handshake with remote engine to exchange metadata.
|
||||
with self._handshake_lock:
|
||||
if remote_engine_id not in self._remote_agents:
|
||||
self._background_nixl_handshake(req_id, remote_engine_id, meta)
|
||||
continue
|
||||
|
||||
# Handshake already completed, start async read xfer.
|
||||
self._read_blocks_for_req(req_id, meta)
|
||||
|
||||
# Start transfers for requests whose handshakes have now finished.
|
||||
while not self._ready_requests.empty():
|
||||
req_id, meta = self._ready_requests.get_nowait()
|
||||
assert meta.remote is not None
|
||||
if meta.remote.engine_id not in self._remote_agents:
|
||||
# The engine was released after its handshake completed, so
|
||||
# handshake again. This fails if the engine is gone.
|
||||
self._background_nixl_handshake(req_id, meta.remote.engine_id, meta)
|
||||
continue
|
||||
self._read_blocks_for_req(req_id, meta)
|
||||
self._read_blocks_for_req(*self._ready_requests.get_nowait())
|
||||
|
||||
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
|
||||
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
|
||||
@@ -151,6 +136,15 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
|
||||
assert meta.remote is not None and self.transfer_topo is not None
|
||||
engine_id = meta.remote.engine_id
|
||||
if engine_id in self._invalid_remote_engines:
|
||||
self._handle_failed_transfer(
|
||||
req_id, None, self._recv_failures, record_failed_transfer=False
|
||||
)
|
||||
return
|
||||
if engine_id not in self._remote_agents:
|
||||
# Also re-handshakes queued requests that outlived invalid-peer cleanup.
|
||||
self._background_nixl_handshake(req_id, engine_id, meta)
|
||||
return
|
||||
# Update last activity from this remote. Mind that cleanup is done on main
|
||||
# thread (this one), so we don't race on this structure.
|
||||
self._engine_last_active[engine_id] = time.perf_counter()
|
||||
|
||||
Reference in New Issue
Block a user