[Bugfix][NIXL] Recover locally invalidated pull peer metadata (#55471)

Signed-off-by: xijiade.aihemaiti <3146335281@qq.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yannik Hinteregger <37209495+YannikHinteregger@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
ahmed xijiaat
2026-10-05 18:46:53 -07:00
committed by GitHub
co-authored by OpenAI Codex Nick Hill Yannik Hinteregger mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9bf4e51030
commit 9465e1c783
5 changed files with 314 additions and 32 deletions
@@ -14,7 +14,7 @@ from collections import defaultdict
from concurrent.futures import Future
from types import SimpleNamespace
from typing import Any, cast
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock, call, patch
import msgspec
import numpy as np
@@ -144,6 +144,7 @@ class FakeNixlWrapper:
def __init__(self, agent_name: str, *args, **kwargs):
self._cycles_before_xfer_done = 0
self._check_xfer_state_cycles: defaultdict[int, int] = defaultdict(lambda: 0)
self.invalid_remote_agents: set[str] = set()
def get_reg_descs(self, caches_data, memory_type: str) -> list:
return [str(uuid.uuid4()) for _ in caches_data]
@@ -164,6 +165,7 @@ class FakeNixlWrapper:
return self.AGENT_METADATA
def add_remote_agent(self, agent_metadata: bytes) -> str:
self.invalid_remote_agents.discard(self.REMOTE_AGENT_NAME)
return self.REMOTE_AGENT_NAME
def get_new_notifs(self) -> dict[str, list[bytes]]:
@@ -185,6 +187,9 @@ class FakeNixlWrapper:
def remove_remote_agent(self, agent: str) -> None:
pass
def check_remote_metadata(self, agent: str) -> bool:
return agent not in self.invalid_remote_agents
def send_notif(self, agent_name: str, notif_msg: bytes) -> None:
pass
@@ -2127,6 +2132,8 @@ def recv_worker():
worker._handshake_futures = {}
worker._remote_agents = {}
worker._engine_by_address = {}
worker._failed_remote_engines = set()
worker._invalid_remote_engines = set()
worker._replicated_pcp_done_sending = set()
worker._invalid_block_ids = queue.Queue()
worker._pending_recv_notifs = {}
@@ -2809,9 +2816,9 @@ class TestPeerReplacement:
self.nixl.REMOTE_AGENT_NAME = engine_id
future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id))
def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True):
def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True, req_id=None):
metadata = NixlConnectorMetadata()
metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta(
metadata.reqs_to_recv[req_id or f"{engine_id}-req"] = ReqMeta(
local_block_ids=(blocks,),
local_physical_block_ids=(blocks,),
tp_size=1,
@@ -2822,6 +2829,11 @@ class TestPeerReplacement:
)
return metadata
def _fail_old_peer(self):
self.worker.start_load_kv(self._request("old"))
self.nixl.invalid_remote_agents.add("old")
self.transport.check_xfer_state.return_value = "ERR"
def test_cleans_old_peer_after_confirmed_handshake(self):
self._connect("healthy", host="other-host")
old_handle = self.worker.dst_xfer_side_handles["old"][0]
@@ -2871,7 +2883,12 @@ class TestPeerReplacement:
],
ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"],
)
def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed):
@pytest.mark.parametrize("invalidated", [False, True])
def test_waits_for_reads_before_cleanup_and_failure_reporting(
self, states, failed, invalidated
):
if invalidated:
self.nixl.invalid_remote_agents.add("old")
self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1]
self.transport.check_xfer_state.side_effect = states
@@ -2890,6 +2907,7 @@ class TestPeerReplacement:
)
assert self.transport.release_xfer_handle.call_count == (2 if failed else 1)
self.transport.remove_remote_agent.assert_called_once_with("old")
assert not self.worker._invalid_remote_engines
def test_waits_for_queued_notification(self):
metadata = self._request("old", blocks=())
@@ -2910,22 +2928,31 @@ class TestPeerReplacement:
assert result.failed_recving == set()
self.transport.remove_remote_agent.assert_called_once_with("old")
def test_waits_for_other_handshakes(self):
@pytest.mark.parametrize("invalidated", [False, True])
def test_waits_for_other_handshakes(self, invalidated):
if invalidated:
self._fail_old_peer()
else:
self._connect()
self.worker._ensure_handshake("other", "other-host", 1234, 1)
self._connect()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_not_called()
self.transport.check_remote_metadata.assert_not_called()
self._connect("other", host="other-host")
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
def test_rehandshakes_queued_request_for_released_engine(self):
@pytest.mark.parametrize("invalidated", [False, True])
def test_rehandshakes_queued_request_for_released_engine(self, invalidated):
# A request queued after its engine's handshake, but drained only after
# that engine was replaced and released, must not read from it.
if invalidated:
self._fail_old_peer()
metadata = self._request("old", blocks=(), awaiting_kvs=False)
self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"]))
self._connect()
if not invalidated:
self._connect()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
@@ -2933,6 +2960,185 @@ class TestPeerReplacement:
assert "old" in self.worker._handshake_futures
self.transport.send_notif.assert_not_called()
@pytest.mark.parametrize("missing", [False, True, RuntimeError("probe failed")])
def test_recovery_requires_confirmed_missing_metadata(self, missing):
self._fail_old_peer()
# A missing rank suffices even if another rank still has metadata.
self.worker._remote_agents["old"][(0, 1)] = "old-other-rank"
self.nixl.invalid_remote_agents.clear()
if missing is True:
self.nixl.invalid_remote_agents.add("old-other-rank")
elif isinstance(missing, Exception):
self.transport.check_remote_metadata.side_effect = missing
result = self.worker.get_transfer_results()
assert result.finished_recving == result.failed_recving == {"old-req"}
assert self.worker.get_block_ids_with_load_errors() == {1}
assert ("old" not in self.worker._remote_agents) is (missing is True)
assert self.transport.release_dlist_handle.call_count == (missing is True)
@pytest.mark.parametrize("handshake_fails", [False, True])
def test_recovery_rehandshakes_same_id_with_fresh_descriptors(
self, handshake_fails
):
old_handle = self.worker.dst_xfer_side_handles["old"][0]
self._fail_old_peer()
self.worker.get_transfer_results()
self.transport.release_dlist_handle.assert_called_once_with(old_handle)
assert not self.worker._engine_by_address
with pytest.raises(KeyError):
self.worker.transfer_topo.get_engine_info("old")
self.transport.make_prepped_xfer.reset_mock()
self.worker.start_load_kv(self._request("old", req_id="next"))
self.transport.make_prepped_xfer.assert_not_called()
if handshake_fails:
self.worker._handshake_futures["old"].set_exception(
RuntimeError("handshake failed")
)
result = self.worker.get_transfer_results()
assert result.failed_recving == result.finished_recving == {"next"}
assert "old" not in self.worker._remote_agents
assert self.worker._ready_requests.empty()
return
self._connect("old")
fresh_handle = self.worker.dst_xfer_side_handles["old"][0]
assert fresh_handle != old_handle
assert self.worker._engine_by_address == {("localhost", 1234): "old"}
self.transport.check_xfer_state.return_value = "DONE"
self.worker.start_load_kv(NixlConnectorMetadata())
assert self.transport.make_prepped_xfer.call_args.args[3] == fresh_handle
result = self.worker.get_transfer_results()
assert result.finished_recving == {"next"} and not result.failed_recving
self.transport.release_dlist_handle.assert_called_once_with(old_handle)
def test_recovery_drains_reads_and_rejects_new_ones_without_harming_other_peers(
self,
):
self._connect("healthy", host="other-host")
self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
self.worker._recving_metadata.update(self._request("healthy").reqs_to_recv)
self.worker._recving_transfers.update(
{"old-req": [10, 11], "healthy-req": [12]}
)
self.nixl.invalid_remote_agents.add("old")
self.transport.check_xfer_state.side_effect = (
lambda h: "ERR" if h == 10 else "PROC"
)
old_handle = self.worker.dst_xfer_side_handles["old"][0]
result = self.worker.get_transfer_results()
assert not result.finished_recving and not result.failed_recving
self.transport.release_dlist_handle.assert_not_called()
self.worker.start_load_kv(self._request("old", req_id="blocked"))
self.transport.make_prepped_xfer.assert_not_called()
self.transport.check_xfer_state.side_effect = (
lambda h: "DONE" if h == 11 else "PROC"
)
result = self.worker.get_transfer_results()
assert (
result.finished_recving == result.failed_recving == {"old-req", "blocked"}
)
assert set(self.worker._remote_agents) == {"healthy"}
assert self.worker._recving_transfers == {"healthy-req": [12]}
self.transport.remove_remote_agent.assert_called_once_with("old")
calls = self.transport.mock_calls
assert calls.index(call.release_xfer_handle(11)) < calls.index(
call.release_dlist_handle(old_handle)
)
def test_repeated_invalidation_backs_off_before_rehandshake(self):
# A peer that is invalidated again soon after recovery (eg a persistent
# RDMA fault) must not re-handshake on every request.
self._fail_old_peer()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
self._connect("old")
self._fail_old_peer()
assert self.worker.get_transfer_results().failed_recving == {"old-req"}
assert "old" in self.worker._remote_agents
self.transport.make_prepped_xfer.reset_mock()
self.worker.start_load_kv(self._request("old", req_id="next"))
assert self.worker.get_transfer_results().failed_recving == {"next"}
self.transport.make_prepped_xfer.assert_not_called()
assert "old" not in self.worker._handshake_futures
self.transport.remove_remote_agent.assert_called_once()
backoff, release_at = self.worker._invalid_engine_backoff["old"]
assert backoff == 1.0
self.worker._invalid_engine_backoff["old"] = (backoff, release_at - backoff)
self.worker.get_transfer_results()
assert self.transport.remove_remote_agent.call_count == 2
assert "old" not in self.worker._remote_agents
def test_recovery_waits_until_failed_handle_can_be_released(self):
self._fail_old_peer()
with patch.object(
self.transport, "release_xfer_handle", side_effect=RuntimeError("in use")
):
result = self.worker.get_transfer_results()
assert not result.finished_recving and not result.failed_recving
assert self.worker._recving_transfers["old-req"]
self.transport.release_dlist_handle.assert_not_called()
result = self.worker.get_transfer_results()
assert result.finished_recving == result.failed_recving == {"old-req"}
self.transport.remove_remote_agent.assert_called_once_with("old")
@pytest.mark.parametrize("expired", [False, True])
def test_success_and_expiry_do_not_probe_metadata(self, expired):
metadata = self._request("old")
if expired:
metadata.reqs_to_recv["old-req"].remote.blocks_expiry_time = 0.0
self.worker._bidirectional_kv_xfer_enabled = True
with (
patch.object(self.worker.xfer_stats, "record_failed_transfer") as failure,
patch.object(self.worker.xfer_stats, "record_kv_expired_req") as expiry,
):
self.worker.start_load_kv(metadata)
result = self.worker.get_transfer_results()
assert result.finished_recving == {"old-req"}
assert result.failed_recving == ({"old-req"} if expired else set())
assert expiry.call_count == int(expired)
failure.assert_not_called()
self.transport.check_remote_metadata.assert_not_called()
self.transport.remove_remote_agent.assert_not_called()
@pytest.mark.parametrize("awaiting_kvs", [False, True])
def test_invalid_peer_empty_recv_is_reported_only_when_awaited(self, awaiting_kvs):
self._fail_old_peer()
self.worker._recving_transfers["old-req"].append(11)
self.transport.check_xfer_state.side_effect = (
lambda h: "PROC" if h == 11 else "ERR"
)
self.worker.get_transfer_results()
self.transport.make_prepped_xfer.reset_mock()
self.worker.start_load_kv(
self._request("old", blocks=(), awaiting_kvs=awaiting_kvs, req_id="empty")
)
result = self.worker.get_transfer_results()
expected = {"empty"} if awaiting_kvs else set()
assert result.finished_recving == result.failed_recving == expected
self.transport.make_prepped_xfer.assert_not_called()
self.transport.send_notif.assert_not_called()
self.transport.remove_remote_agent.assert_not_called()
def test_eviction_clears_pending_recovery(self):
self.worker._failed_remote_engines.add("old")
self.worker._invalid_remote_engines.add("old")
self.worker._engine_ttl = 1
self.worker._engine_last_active["old"] = time.perf_counter() - 10
self.worker._evict_stale_engines()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
assert not self.worker._failed_remote_engines
assert not self.worker._invalid_remote_engines
def test_transfer_topology_unregister():
"""TransferTopology.unregister_remote_engine removes the engine."""
@@ -60,6 +60,8 @@ def region_pull_worker():
worker._transfer_layer_group_ids = ()
worker._bidirectional_kv_xfer_enabled = False
worker._recving_transfers = {}
worker._failed_remote_engines = set()
worker._invalid_remote_engines = set()
worker.use_mla, worker._has_mamba = True, False
worker.dcp_size = 1
worker.dcp_rank = 0
@@ -690,6 +692,8 @@ def test_read_blocks_for_req_expands_remote_ids(
worker = object.__new__(NixlConnectorWorker)
worker._physical_blocks_per_logical_kv_block = local_physical_per_logical
worker._invalid_remote_engines = set()
worker._remote_agents = {"remote-engine": {}}
worker._engine_last_active = {}
worker._recving_transfers = {}
worker._bidirectional_kv_xfer_enabled = False
@@ -379,6 +379,8 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w.pp_size = 1
w.engine_id = "test-decode-engine"
w._remote_agents = {}
w._failed_remote_engines = set()
w._invalid_remote_engines = set()
w._handshake_lock = threading.RLock()
w._physical_blocks_per_logical_kv_block = 1
w._uses_region_group_mapping = False