[Bugfix][NIXL] Recover locally invalidated pull peer metadata (#55471)

Signed-off-by: xijiade.aihemaiti <3146335281@qq.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yannik Hinteregger <37209495+YannikHinteregger@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
ahmed xijiaat
2026-10-05 18:46:53 -07:00
committed by GitHub
co-authored by OpenAI Codex Nick Hill Yannik Hinteregger mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9bf4e51030
commit 9465e1c783
5 changed files with 314 additions and 32 deletions
@@ -101,6 +101,11 @@ logger = init_logger(__name__)
_SHARED_REGION_GROUP_ID = -1
# Re-handshake delay bounds for a pull peer whose local metadata keeps getting
# invalidated, eg due to a persistent RDMA fault.
_INVALID_PEER_MIN_BACKOFF_S = 1.0
_INVALID_PEER_MAX_BACKOFF_S = 60.0
def _region_sort_key(layer_name: str) -> tuple[tuple[int, int | str], ...]:
"""Sort transfer regions in model-layer order, then by cache name."""
@@ -685,6 +690,12 @@ class NixlBaseConnectorWorker:
self._remote_agents: dict[EngineId, dict[tuple[int, int], str]] = defaultdict(
dict
)
# Failed reads need a local metadata probe; invalid peers wait for drain.
self._failed_remote_engines: set[EngineId] = set()
self._invalid_remote_engines: set[EngineId] = set()
# Invalid engine -> (backoff, earliest release time). Kept across release
# so that repeat invalidations back off.
self._invalid_engine_backoff: dict[EngineId, tuple[float, float]] = {}
# Map of engine_id -> clock offset.
self._engine_clock_offset: dict[EngineId, float] = {}
@@ -3029,7 +3040,7 @@ class NixlBaseConnectorWorker:
# Handle timeout to avoid stranding blocks on remote.
self._reap_expired_send_leases(done_sending)
self._cleanup_replaced_remote_engines()
self._cleanup_remote_engines()
return KVConnectorTransferResults(
finished_sending=done_sending,
finished_recving=done_recving,
@@ -3195,6 +3206,12 @@ class NixlBaseConnectorWorker:
"""
if record_failed_transfer:
self.xfer_stats.record_failed_transfer()
if (
self._TRANSFER_MODE == "pull"
and (meta := self._recving_metadata.get(req_id)) is not None
and meta.remote is not None
):
self._failed_remote_engines.add(meta.remote.engine_id)
if failed_req_ids is not None:
failed_req_ids.add(req_id)
return handle is None or self._try_release_xfer_handle(req_id, handle)
@@ -3642,15 +3659,38 @@ class NixlBaseConnectorWorker:
engine_id,
)
def _cleanup_replaced_remote_engines(self) -> None:
"""Release replaced pull peers once requests and handshakes have drained."""
def _cleanup_remote_engines(self) -> None:
"""Release replaced or locally invalid pull peers after their reads drain."""
# TODO: Also handle push mode, which handshakes in both directions.
if self._TRANSFER_MODE != "pull":
return
# A background handshake may be loading native metadata with the GIL
# released. Defer both probing and cleanup until it has finished.
with self._handshake_lock:
if self._handshake_futures:
return
replaced = self._remote_agents.keys() - self._engine_by_address.values()
if not replaced:
now = time.perf_counter()
for engine_id in self._failed_remote_engines - self._invalid_remote_engines:
agents = self._remote_agents.get(engine_id, {}).values()
try:
# No descriptors: query local existence, not peer health.
if not all(map(self.nixl_wrapper.check_remote_metadata, agents)):
self._mark_remote_engine_invalid(engine_id, now)
except Exception:
logger.warning(
"Could not check local NIXL metadata for engine %s.",
engine_id,
exc_info=True,
)
self._failed_remote_engines.clear()
# Invalid peers keep rejecting new reads until their backoff has elapsed.
invalid = {
engine_id
for engine_id in self._invalid_remote_engines
if self._invalid_engine_backoff.get(engine_id, (0.0, 0.0))[1] <= now
}
if not replaced and not invalid:
return
# _recving_metadata is only accessed from this (main) thread.
busy = {
@@ -3658,9 +3698,42 @@ class NixlBaseConnectorWorker:
for meta in self._recving_metadata.values()
if meta.remote is not None
}
for engine_id in replaced - busy:
# Replaced peers can still serve queued requests. Invalid peers reject
# new reads, but must retain descriptors until outstanding handles end.
invalid -= self._engines_with_inflight_transfers()
for engine_id in (replaced - busy) | invalid:
self._cleanup_remote_engine(engine_id, log_eviction=False)
logger.info("Released NIXL state for replaced remote engine %s.", engine_id)
logger.info(
"Released NIXL state for replaced or locally invalid remote engine %s.",
engine_id,
)
def _mark_remote_engine_invalid(self, engine_id: EngineId, now: float) -> None:
"""Schedule release of an engine whose local NIXL metadata is gone.
The first invalidation is released as soon as reads drain. Repeats
before the peer has stayed healthy for the max backoff double the
delay, during which reads fail fast instead of re-handshaking.
"""
# Forget engines that have stayed healthy long enough.
for eid, (_, release_at) in list(self._invalid_engine_backoff.items()):
if now - release_at > _INVALID_PEER_MAX_BACKOFF_S:
del self._invalid_engine_backoff[eid]
prev = self._invalid_engine_backoff.get(engine_id)
backoff = 0.0
if prev is not None:
backoff = min(
max(2 * prev[0], _INVALID_PEER_MIN_BACKOFF_S),
_INVALID_PEER_MAX_BACKOFF_S,
)
self._invalid_engine_backoff[engine_id] = (backoff, now + backoff)
self._invalid_remote_engines.add(engine_id)
logger.warning(
"Local NIXL metadata for remote engine %s was invalidated; "
"re-handshaking after reads drain%s.",
engine_id,
f" and a {backoff:.0f}s backoff" if backoff else "",
)
def _cleanup_remote_engine(
self, engine_id: EngineId, *, log_eviction: bool = True
@@ -3668,8 +3741,8 @@ class NixlBaseConnectorWorker:
"""Remove all state for a single remote engine.
Releases NIXL resources (dlist handles, remote agents) and clears
all per-engine data structures. Used by TTL eviction, replaced peer
cleanup and shutdown.
all per-engine data structures. Used by TTL eviction, replaced/invalid
peer cleanup and shutdown.
"""
assert engine_id in self._remote_agents
@@ -3685,6 +3758,9 @@ class NixlBaseConnectorWorker:
for agent_name in agents.values():
self.nixl_wrapper.remove_remote_agent(agent_name)
self._failed_remote_engines.discard(engine_id)
self._invalid_remote_engines.discard(engine_id)
self.kv_caches_base_addr.pop(engine_id, None)
self.dst_num_blocks.pop(engine_id, None)
self.dst_region_num_blocks.pop(engine_id, None)
@@ -72,26 +72,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
# Aborted cleanup requests have already been removed from the scheduler.
if meta.awaiting_kvs or any(meta.local_block_ids):
self._recving_metadata[req_id] = meta
if remote_engine_id not in self._remote_agents:
# Initiate handshake with remote engine to exchange metadata.
with self._handshake_lock:
if remote_engine_id not in self._remote_agents:
self._background_nixl_handshake(req_id, remote_engine_id, meta)
continue
# Handshake already completed, start async read xfer.
self._read_blocks_for_req(req_id, meta)
# Start transfers for requests whose handshakes have now finished.
while not self._ready_requests.empty():
req_id, meta = self._ready_requests.get_nowait()
assert meta.remote is not None
if meta.remote.engine_id not in self._remote_agents:
# The engine was released after its handshake completed, so
# handshake again. This fails if the engine is gone.
self._background_nixl_handshake(req_id, meta.remote.engine_id, meta)
continue
self._read_blocks_for_req(req_id, meta)
self._read_blocks_for_req(*self._ready_requests.get_nowait())
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
@@ -151,6 +136,15 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
assert meta.remote is not None and self.transfer_topo is not None
engine_id = meta.remote.engine_id
if engine_id in self._invalid_remote_engines:
self._handle_failed_transfer(
req_id, None, self._recv_failures, record_failed_transfer=False
)
return
if engine_id not in self._remote_agents:
# Also re-handshakes queued requests that outlived invalid-peer cleanup.
self._background_nixl_handshake(req_id, engine_id, meta)
return
# Update last activity from this remote. Mind that cleanup is done on main
# thread (this one), so we don't race on this structure.
self._engine_last_active[engine_id] = time.perf_counter()