[Bugfix][NIXL] Recover locally invalidated pull peer metadata (#55471)

Signed-off-by: xijiade.aihemaiti <3146335281@qq.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yannik Hinteregger <37209495+YannikHinteregger@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
ahmed xijiaat
2026-10-05 18:46:53 -07:00
committed by GitHub
co-authored by OpenAI Codex Nick Hill Yannik Hinteregger mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9bf4e51030
commit 9465e1c783
5 changed files with 314 additions and 32 deletions
@@ -14,7 +14,7 @@ from collections import defaultdict
from concurrent.futures import Future
from types import SimpleNamespace
from typing import Any, cast
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock, call, patch
import msgspec
import numpy as np
@@ -144,6 +144,7 @@ class FakeNixlWrapper:
def __init__(self, agent_name: str, *args, **kwargs):
self._cycles_before_xfer_done = 0
self._check_xfer_state_cycles: defaultdict[int, int] = defaultdict(lambda: 0)
self.invalid_remote_agents: set[str] = set()
def get_reg_descs(self, caches_data, memory_type: str) -> list:
return [str(uuid.uuid4()) for _ in caches_data]
@@ -164,6 +165,7 @@ class FakeNixlWrapper:
return self.AGENT_METADATA
def add_remote_agent(self, agent_metadata: bytes) -> str:
self.invalid_remote_agents.discard(self.REMOTE_AGENT_NAME)
return self.REMOTE_AGENT_NAME
def get_new_notifs(self) -> dict[str, list[bytes]]:
@@ -185,6 +187,9 @@ class FakeNixlWrapper:
def remove_remote_agent(self, agent: str) -> None:
pass
def check_remote_metadata(self, agent: str) -> bool:
return agent not in self.invalid_remote_agents
def send_notif(self, agent_name: str, notif_msg: bytes) -> None:
pass
@@ -2127,6 +2132,8 @@ def recv_worker():
worker._handshake_futures = {}
worker._remote_agents = {}
worker._engine_by_address = {}
worker._failed_remote_engines = set()
worker._invalid_remote_engines = set()
worker._replicated_pcp_done_sending = set()
worker._invalid_block_ids = queue.Queue()
worker._pending_recv_notifs = {}
@@ -2809,9 +2816,9 @@ class TestPeerReplacement:
self.nixl.REMOTE_AGENT_NAME = engine_id
future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id))
def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True):
def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True, req_id=None):
metadata = NixlConnectorMetadata()
metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta(
metadata.reqs_to_recv[req_id or f"{engine_id}-req"] = ReqMeta(
local_block_ids=(blocks,),
local_physical_block_ids=(blocks,),
tp_size=1,
@@ -2822,6 +2829,11 @@ class TestPeerReplacement:
)
return metadata
def _fail_old_peer(self):
self.worker.start_load_kv(self._request("old"))
self.nixl.invalid_remote_agents.add("old")
self.transport.check_xfer_state.return_value = "ERR"
def test_cleans_old_peer_after_confirmed_handshake(self):
self._connect("healthy", host="other-host")
old_handle = self.worker.dst_xfer_side_handles["old"][0]
@@ -2871,7 +2883,12 @@ class TestPeerReplacement:
],
ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"],
)
def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed):
@pytest.mark.parametrize("invalidated", [False, True])
def test_waits_for_reads_before_cleanup_and_failure_reporting(
self, states, failed, invalidated
):
if invalidated:
self.nixl.invalid_remote_agents.add("old")
self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1]
self.transport.check_xfer_state.side_effect = states
@@ -2890,6 +2907,7 @@ class TestPeerReplacement:
)
assert self.transport.release_xfer_handle.call_count == (2 if failed else 1)
self.transport.remove_remote_agent.assert_called_once_with("old")
assert not self.worker._invalid_remote_engines
def test_waits_for_queued_notification(self):
metadata = self._request("old", blocks=())
@@ -2910,22 +2928,31 @@ class TestPeerReplacement:
assert result.failed_recving == set()
self.transport.remove_remote_agent.assert_called_once_with("old")
def test_waits_for_other_handshakes(self):
@pytest.mark.parametrize("invalidated", [False, True])
def test_waits_for_other_handshakes(self, invalidated):
if invalidated:
self._fail_old_peer()
else:
self._connect()
self.worker._ensure_handshake("other", "other-host", 1234, 1)
self._connect()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_not_called()
self.transport.check_remote_metadata.assert_not_called()
self._connect("other", host="other-host")
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
def test_rehandshakes_queued_request_for_released_engine(self):
@pytest.mark.parametrize("invalidated", [False, True])
def test_rehandshakes_queued_request_for_released_engine(self, invalidated):
# A request queued after its engine's handshake, but drained only after
# that engine was replaced and released, must not read from it.
if invalidated:
self._fail_old_peer()
metadata = self._request("old", blocks=(), awaiting_kvs=False)
self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"]))
self._connect()
if not invalidated:
self._connect()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
@@ -2933,6 +2960,185 @@ class TestPeerReplacement:
assert "old" in self.worker._handshake_futures
self.transport.send_notif.assert_not_called()
@pytest.mark.parametrize("missing", [False, True, RuntimeError("probe failed")])
def test_recovery_requires_confirmed_missing_metadata(self, missing):
self._fail_old_peer()
# A missing rank suffices even if another rank still has metadata.
self.worker._remote_agents["old"][(0, 1)] = "old-other-rank"
self.nixl.invalid_remote_agents.clear()
if missing is True:
self.nixl.invalid_remote_agents.add("old-other-rank")
elif isinstance(missing, Exception):
self.transport.check_remote_metadata.side_effect = missing
result = self.worker.get_transfer_results()
assert result.finished_recving == result.failed_recving == {"old-req"}
assert self.worker.get_block_ids_with_load_errors() == {1}
assert ("old" not in self.worker._remote_agents) is (missing is True)
assert self.transport.release_dlist_handle.call_count == (missing is True)
@pytest.mark.parametrize("handshake_fails", [False, True])
def test_recovery_rehandshakes_same_id_with_fresh_descriptors(
self, handshake_fails
):
old_handle = self.worker.dst_xfer_side_handles["old"][0]
self._fail_old_peer()
self.worker.get_transfer_results()
self.transport.release_dlist_handle.assert_called_once_with(old_handle)
assert not self.worker._engine_by_address
with pytest.raises(KeyError):
self.worker.transfer_topo.get_engine_info("old")
self.transport.make_prepped_xfer.reset_mock()
self.worker.start_load_kv(self._request("old", req_id="next"))
self.transport.make_prepped_xfer.assert_not_called()
if handshake_fails:
self.worker._handshake_futures["old"].set_exception(
RuntimeError("handshake failed")
)
result = self.worker.get_transfer_results()
assert result.failed_recving == result.finished_recving == {"next"}
assert "old" not in self.worker._remote_agents
assert self.worker._ready_requests.empty()
return
self._connect("old")
fresh_handle = self.worker.dst_xfer_side_handles["old"][0]
assert fresh_handle != old_handle
assert self.worker._engine_by_address == {("localhost", 1234): "old"}
self.transport.check_xfer_state.return_value = "DONE"
self.worker.start_load_kv(NixlConnectorMetadata())
assert self.transport.make_prepped_xfer.call_args.args[3] == fresh_handle
result = self.worker.get_transfer_results()
assert result.finished_recving == {"next"} and not result.failed_recving
self.transport.release_dlist_handle.assert_called_once_with(old_handle)
def test_recovery_drains_reads_and_rejects_new_ones_without_harming_other_peers(
self,
):
self._connect("healthy", host="other-host")
self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
self.worker._recving_metadata.update(self._request("healthy").reqs_to_recv)
self.worker._recving_transfers.update(
{"old-req": [10, 11], "healthy-req": [12]}
)
self.nixl.invalid_remote_agents.add("old")
self.transport.check_xfer_state.side_effect = (
lambda h: "ERR" if h == 10 else "PROC"
)
old_handle = self.worker.dst_xfer_side_handles["old"][0]
result = self.worker.get_transfer_results()
assert not result.finished_recving and not result.failed_recving
self.transport.release_dlist_handle.assert_not_called()
self.worker.start_load_kv(self._request("old", req_id="blocked"))
self.transport.make_prepped_xfer.assert_not_called()
self.transport.check_xfer_state.side_effect = (
lambda h: "DONE" if h == 11 else "PROC"
)
result = self.worker.get_transfer_results()
assert (
result.finished_recving == result.failed_recving == {"old-req", "blocked"}
)
assert set(self.worker._remote_agents) == {"healthy"}
assert self.worker._recving_transfers == {"healthy-req": [12]}
self.transport.remove_remote_agent.assert_called_once_with("old")
calls = self.transport.mock_calls
assert calls.index(call.release_xfer_handle(11)) < calls.index(
call.release_dlist_handle(old_handle)
)
def test_repeated_invalidation_backs_off_before_rehandshake(self):
# A peer that is invalidated again soon after recovery (eg a persistent
# RDMA fault) must not re-handshake on every request.
self._fail_old_peer()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
self._connect("old")
self._fail_old_peer()
assert self.worker.get_transfer_results().failed_recving == {"old-req"}
assert "old" in self.worker._remote_agents
self.transport.make_prepped_xfer.reset_mock()
self.worker.start_load_kv(self._request("old", req_id="next"))
assert self.worker.get_transfer_results().failed_recving == {"next"}
self.transport.make_prepped_xfer.assert_not_called()
assert "old" not in self.worker._handshake_futures
self.transport.remove_remote_agent.assert_called_once()
backoff, release_at = self.worker._invalid_engine_backoff["old"]
assert backoff == 1.0
self.worker._invalid_engine_backoff["old"] = (backoff, release_at - backoff)
self.worker.get_transfer_results()
assert self.transport.remove_remote_agent.call_count == 2
assert "old" not in self.worker._remote_agents
def test_recovery_waits_until_failed_handle_can_be_released(self):
self._fail_old_peer()
with patch.object(
self.transport, "release_xfer_handle", side_effect=RuntimeError("in use")
):
result = self.worker.get_transfer_results()
assert not result.finished_recving and not result.failed_recving
assert self.worker._recving_transfers["old-req"]
self.transport.release_dlist_handle.assert_not_called()
result = self.worker.get_transfer_results()
assert result.finished_recving == result.failed_recving == {"old-req"}
self.transport.remove_remote_agent.assert_called_once_with("old")
@pytest.mark.parametrize("expired", [False, True])
def test_success_and_expiry_do_not_probe_metadata(self, expired):
metadata = self._request("old")
if expired:
metadata.reqs_to_recv["old-req"].remote.blocks_expiry_time = 0.0
self.worker._bidirectional_kv_xfer_enabled = True
with (
patch.object(self.worker.xfer_stats, "record_failed_transfer") as failure,
patch.object(self.worker.xfer_stats, "record_kv_expired_req") as expiry,
):
self.worker.start_load_kv(metadata)
result = self.worker.get_transfer_results()
assert result.finished_recving == {"old-req"}
assert result.failed_recving == ({"old-req"} if expired else set())
assert expiry.call_count == int(expired)
failure.assert_not_called()
self.transport.check_remote_metadata.assert_not_called()
self.transport.remove_remote_agent.assert_not_called()
@pytest.mark.parametrize("awaiting_kvs", [False, True])
def test_invalid_peer_empty_recv_is_reported_only_when_awaited(self, awaiting_kvs):
self._fail_old_peer()
self.worker._recving_transfers["old-req"].append(11)
self.transport.check_xfer_state.side_effect = (
lambda h: "PROC" if h == 11 else "ERR"
)
self.worker.get_transfer_results()
self.transport.make_prepped_xfer.reset_mock()
self.worker.start_load_kv(
self._request("old", blocks=(), awaiting_kvs=awaiting_kvs, req_id="empty")
)
result = self.worker.get_transfer_results()
expected = {"empty"} if awaiting_kvs else set()
assert result.finished_recving == result.failed_recving == expected
self.transport.make_prepped_xfer.assert_not_called()
self.transport.send_notif.assert_not_called()
self.transport.remove_remote_agent.assert_not_called()
def test_eviction_clears_pending_recovery(self):
self.worker._failed_remote_engines.add("old")
self.worker._invalid_remote_engines.add("old")
self.worker._engine_ttl = 1
self.worker._engine_last_active["old"] = time.perf_counter() - 10
self.worker._evict_stale_engines()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
assert not self.worker._failed_remote_engines
assert not self.worker._invalid_remote_engines
def test_transfer_topology_unregister():
"""TransferTopology.unregister_remote_engine removes the engine."""
@@ -60,6 +60,8 @@ def region_pull_worker():
worker._transfer_layer_group_ids = ()
worker._bidirectional_kv_xfer_enabled = False
worker._recving_transfers = {}
worker._failed_remote_engines = set()
worker._invalid_remote_engines = set()
worker.use_mla, worker._has_mamba = True, False
worker.dcp_size = 1
worker.dcp_rank = 0
@@ -690,6 +692,8 @@ def test_read_blocks_for_req_expands_remote_ids(
worker = object.__new__(NixlConnectorWorker)
worker._physical_blocks_per_logical_kv_block = local_physical_per_logical
worker._invalid_remote_engines = set()
worker._remote_agents = {"remote-engine": {}}
worker._engine_last_active = {}
worker._recving_transfers = {}
worker._bidirectional_kv_xfer_enabled = False
@@ -379,6 +379,8 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w.pp_size = 1
w.engine_id = "test-decode-engine"
w._remote_agents = {}
w._failed_remote_engines = set()
w._invalid_remote_engines = set()
w._handshake_lock = threading.RLock()
w._physical_blocks_per_logical_kv_block = 1
w._uses_region_group_mapping = False
@@ -101,6 +101,11 @@ logger = init_logger(__name__)
_SHARED_REGION_GROUP_ID = -1
# Re-handshake delay bounds for a pull peer whose local metadata keeps getting
# invalidated, eg due to a persistent RDMA fault.
_INVALID_PEER_MIN_BACKOFF_S = 1.0
_INVALID_PEER_MAX_BACKOFF_S = 60.0
def _region_sort_key(layer_name: str) -> tuple[tuple[int, int | str], ...]:
"""Sort transfer regions in model-layer order, then by cache name."""
@@ -685,6 +690,12 @@ class NixlBaseConnectorWorker:
self._remote_agents: dict[EngineId, dict[tuple[int, int], str]] = defaultdict(
dict
)
# Failed reads need a local metadata probe; invalid peers wait for drain.
self._failed_remote_engines: set[EngineId] = set()
self._invalid_remote_engines: set[EngineId] = set()
# Invalid engine -> (backoff, earliest release time). Kept across release
# so that repeat invalidations back off.
self._invalid_engine_backoff: dict[EngineId, tuple[float, float]] = {}
# Map of engine_id -> clock offset.
self._engine_clock_offset: dict[EngineId, float] = {}
@@ -3029,7 +3040,7 @@ class NixlBaseConnectorWorker:
# Handle timeout to avoid stranding blocks on remote.
self._reap_expired_send_leases(done_sending)
self._cleanup_replaced_remote_engines()
self._cleanup_remote_engines()
return KVConnectorTransferResults(
finished_sending=done_sending,
finished_recving=done_recving,
@@ -3195,6 +3206,12 @@ class NixlBaseConnectorWorker:
"""
if record_failed_transfer:
self.xfer_stats.record_failed_transfer()
if (
self._TRANSFER_MODE == "pull"
and (meta := self._recving_metadata.get(req_id)) is not None
and meta.remote is not None
):
self._failed_remote_engines.add(meta.remote.engine_id)
if failed_req_ids is not None:
failed_req_ids.add(req_id)
return handle is None or self._try_release_xfer_handle(req_id, handle)
@@ -3642,15 +3659,38 @@ class NixlBaseConnectorWorker:
engine_id,
)
def _cleanup_replaced_remote_engines(self) -> None:
"""Release replaced pull peers once requests and handshakes have drained."""
def _cleanup_remote_engines(self) -> None:
"""Release replaced or locally invalid pull peers after their reads drain."""
# TODO: Also handle push mode, which handshakes in both directions.
if self._TRANSFER_MODE != "pull":
return
# A background handshake may be loading native metadata with the GIL
# released. Defer both probing and cleanup until it has finished.
with self._handshake_lock:
if self._handshake_futures:
return
replaced = self._remote_agents.keys() - self._engine_by_address.values()
if not replaced:
now = time.perf_counter()
for engine_id in self._failed_remote_engines - self._invalid_remote_engines:
agents = self._remote_agents.get(engine_id, {}).values()
try:
# No descriptors: query local existence, not peer health.
if not all(map(self.nixl_wrapper.check_remote_metadata, agents)):
self._mark_remote_engine_invalid(engine_id, now)
except Exception:
logger.warning(
"Could not check local NIXL metadata for engine %s.",
engine_id,
exc_info=True,
)
self._failed_remote_engines.clear()
# Invalid peers keep rejecting new reads until their backoff has elapsed.
invalid = {
engine_id
for engine_id in self._invalid_remote_engines
if self._invalid_engine_backoff.get(engine_id, (0.0, 0.0))[1] <= now
}
if not replaced and not invalid:
return
# _recving_metadata is only accessed from this (main) thread.
busy = {
@@ -3658,9 +3698,42 @@ class NixlBaseConnectorWorker:
for meta in self._recving_metadata.values()
if meta.remote is not None
}
for engine_id in replaced - busy:
# Replaced peers can still serve queued requests. Invalid peers reject
# new reads, but must retain descriptors until outstanding handles end.
invalid -= self._engines_with_inflight_transfers()
for engine_id in (replaced - busy) | invalid:
self._cleanup_remote_engine(engine_id, log_eviction=False)
logger.info("Released NIXL state for replaced remote engine %s.", engine_id)
logger.info(
"Released NIXL state for replaced or locally invalid remote engine %s.",
engine_id,
)
def _mark_remote_engine_invalid(self, engine_id: EngineId, now: float) -> None:
"""Schedule release of an engine whose local NIXL metadata is gone.
The first invalidation is released as soon as reads drain. Repeats
before the peer has stayed healthy for the max backoff double the
delay, during which reads fail fast instead of re-handshaking.
"""
# Forget engines that have stayed healthy long enough.
for eid, (_, release_at) in list(self._invalid_engine_backoff.items()):
if now - release_at > _INVALID_PEER_MAX_BACKOFF_S:
del self._invalid_engine_backoff[eid]
prev = self._invalid_engine_backoff.get(engine_id)
backoff = 0.0
if prev is not None:
backoff = min(
max(2 * prev[0], _INVALID_PEER_MIN_BACKOFF_S),
_INVALID_PEER_MAX_BACKOFF_S,
)
self._invalid_engine_backoff[engine_id] = (backoff, now + backoff)
self._invalid_remote_engines.add(engine_id)
logger.warning(
"Local NIXL metadata for remote engine %s was invalidated; "
"re-handshaking after reads drain%s.",
engine_id,
f" and a {backoff:.0f}s backoff" if backoff else "",
)
def _cleanup_remote_engine(
self, engine_id: EngineId, *, log_eviction: bool = True
@@ -3668,8 +3741,8 @@ class NixlBaseConnectorWorker:
"""Remove all state for a single remote engine.
Releases NIXL resources (dlist handles, remote agents) and clears
all per-engine data structures. Used by TTL eviction, replaced peer
cleanup and shutdown.
all per-engine data structures. Used by TTL eviction, replaced/invalid
peer cleanup and shutdown.
"""
assert engine_id in self._remote_agents
@@ -3685,6 +3758,9 @@ class NixlBaseConnectorWorker:
for agent_name in agents.values():
self.nixl_wrapper.remove_remote_agent(agent_name)
self._failed_remote_engines.discard(engine_id)
self._invalid_remote_engines.discard(engine_id)
self.kv_caches_base_addr.pop(engine_id, None)
self.dst_num_blocks.pop(engine_id, None)
self.dst_region_num_blocks.pop(engine_id, None)
@@ -72,26 +72,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
# Aborted cleanup requests have already been removed from the scheduler.
if meta.awaiting_kvs or any(meta.local_block_ids):
self._recving_metadata[req_id] = meta
if remote_engine_id not in self._remote_agents:
# Initiate handshake with remote engine to exchange metadata.
with self._handshake_lock:
if remote_engine_id not in self._remote_agents:
self._background_nixl_handshake(req_id, remote_engine_id, meta)
continue
# Handshake already completed, start async read xfer.
self._read_blocks_for_req(req_id, meta)
# Start transfers for requests whose handshakes have now finished.
while not self._ready_requests.empty():
req_id, meta = self._ready_requests.get_nowait()
assert meta.remote is not None
if meta.remote.engine_id not in self._remote_agents:
# The engine was released after its handshake completed, so
# handshake again. This fails if the engine is gone.
self._background_nixl_handshake(req_id, meta.remote.engine_id, meta)
continue
self._read_blocks_for_req(req_id, meta)
self._read_blocks_for_req(*self._ready_requests.get_nowait())
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
@@ -151,6 +136,15 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
assert meta.remote is not None and self.transfer_topo is not None
engine_id = meta.remote.engine_id
if engine_id in self._invalid_remote_engines:
self._handle_failed_transfer(
req_id, None, self._recv_failures, record_failed_transfer=False
)
return
if engine_id not in self._remote_agents:
# Also re-handshakes queued requests that outlived invalid-peer cleanup.
self._background_nixl_handshake(req_id, engine_id, meta)
return
# Update last activity from this remote. Mind that cleanup is done on main
# thread (this one), so we don't race on this structure.
self._engine_last_active[engine_id] = time.perf_counter()