[Bugfix][NIXL] Recover locally invalidated pull peer metadata (#55471)
Signed-off-by: xijiade.aihemaiti <3146335281@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yannik Hinteregger <37209495+YannikHinteregger@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Nick Hill
Yannik Hinteregger
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9bf4e51030
commit
9465e1c783
@@ -14,7 +14,7 @@ from collections import defaultdict
|
||||
from concurrent.futures import Future
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
@@ -144,6 +144,7 @@ class FakeNixlWrapper:
|
||||
def __init__(self, agent_name: str, *args, **kwargs):
|
||||
self._cycles_before_xfer_done = 0
|
||||
self._check_xfer_state_cycles: defaultdict[int, int] = defaultdict(lambda: 0)
|
||||
self.invalid_remote_agents: set[str] = set()
|
||||
|
||||
def get_reg_descs(self, caches_data, memory_type: str) -> list:
|
||||
return [str(uuid.uuid4()) for _ in caches_data]
|
||||
@@ -164,6 +165,7 @@ class FakeNixlWrapper:
|
||||
return self.AGENT_METADATA
|
||||
|
||||
def add_remote_agent(self, agent_metadata: bytes) -> str:
|
||||
self.invalid_remote_agents.discard(self.REMOTE_AGENT_NAME)
|
||||
return self.REMOTE_AGENT_NAME
|
||||
|
||||
def get_new_notifs(self) -> dict[str, list[bytes]]:
|
||||
@@ -185,6 +187,9 @@ class FakeNixlWrapper:
|
||||
def remove_remote_agent(self, agent: str) -> None:
|
||||
pass
|
||||
|
||||
def check_remote_metadata(self, agent: str) -> bool:
|
||||
return agent not in self.invalid_remote_agents
|
||||
|
||||
def send_notif(self, agent_name: str, notif_msg: bytes) -> None:
|
||||
pass
|
||||
|
||||
@@ -2127,6 +2132,8 @@ def recv_worker():
|
||||
worker._handshake_futures = {}
|
||||
worker._remote_agents = {}
|
||||
worker._engine_by_address = {}
|
||||
worker._failed_remote_engines = set()
|
||||
worker._invalid_remote_engines = set()
|
||||
worker._replicated_pcp_done_sending = set()
|
||||
worker._invalid_block_ids = queue.Queue()
|
||||
worker._pending_recv_notifs = {}
|
||||
@@ -2809,9 +2816,9 @@ class TestPeerReplacement:
|
||||
self.nixl.REMOTE_AGENT_NAME = engine_id
|
||||
future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id))
|
||||
|
||||
def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True):
|
||||
def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True, req_id=None):
|
||||
metadata = NixlConnectorMetadata()
|
||||
metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta(
|
||||
metadata.reqs_to_recv[req_id or f"{engine_id}-req"] = ReqMeta(
|
||||
local_block_ids=(blocks,),
|
||||
local_physical_block_ids=(blocks,),
|
||||
tp_size=1,
|
||||
@@ -2822,6 +2829,11 @@ class TestPeerReplacement:
|
||||
)
|
||||
return metadata
|
||||
|
||||
def _fail_old_peer(self):
|
||||
self.worker.start_load_kv(self._request("old"))
|
||||
self.nixl.invalid_remote_agents.add("old")
|
||||
self.transport.check_xfer_state.return_value = "ERR"
|
||||
|
||||
def test_cleans_old_peer_after_confirmed_handshake(self):
|
||||
self._connect("healthy", host="other-host")
|
||||
old_handle = self.worker.dst_xfer_side_handles["old"][0]
|
||||
@@ -2871,7 +2883,12 @@ class TestPeerReplacement:
|
||||
],
|
||||
ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"],
|
||||
)
|
||||
def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed):
|
||||
@pytest.mark.parametrize("invalidated", [False, True])
|
||||
def test_waits_for_reads_before_cleanup_and_failure_reporting(
|
||||
self, states, failed, invalidated
|
||||
):
|
||||
if invalidated:
|
||||
self.nixl.invalid_remote_agents.add("old")
|
||||
self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
|
||||
self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1]
|
||||
self.transport.check_xfer_state.side_effect = states
|
||||
@@ -2890,6 +2907,7 @@ class TestPeerReplacement:
|
||||
)
|
||||
assert self.transport.release_xfer_handle.call_count == (2 if failed else 1)
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
assert not self.worker._invalid_remote_engines
|
||||
|
||||
def test_waits_for_queued_notification(self):
|
||||
metadata = self._request("old", blocks=())
|
||||
@@ -2910,22 +2928,31 @@ class TestPeerReplacement:
|
||||
assert result.failed_recving == set()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
def test_waits_for_other_handshakes(self):
|
||||
@pytest.mark.parametrize("invalidated", [False, True])
|
||||
def test_waits_for_other_handshakes(self, invalidated):
|
||||
if invalidated:
|
||||
self._fail_old_peer()
|
||||
else:
|
||||
self._connect()
|
||||
self.worker._ensure_handshake("other", "other-host", 1234, 1)
|
||||
self._connect()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
self.transport.check_remote_metadata.assert_not_called()
|
||||
|
||||
self._connect("other", host="other-host")
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
def test_rehandshakes_queued_request_for_released_engine(self):
|
||||
@pytest.mark.parametrize("invalidated", [False, True])
|
||||
def test_rehandshakes_queued_request_for_released_engine(self, invalidated):
|
||||
# A request queued after its engine's handshake, but drained only after
|
||||
# that engine was replaced and released, must not read from it.
|
||||
if invalidated:
|
||||
self._fail_old_peer()
|
||||
metadata = self._request("old", blocks=(), awaiting_kvs=False)
|
||||
self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"]))
|
||||
self._connect()
|
||||
if not invalidated:
|
||||
self._connect()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
@@ -2933,6 +2960,185 @@ class TestPeerReplacement:
|
||||
assert "old" in self.worker._handshake_futures
|
||||
self.transport.send_notif.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize("missing", [False, True, RuntimeError("probe failed")])
|
||||
def test_recovery_requires_confirmed_missing_metadata(self, missing):
|
||||
self._fail_old_peer()
|
||||
# A missing rank suffices even if another rank still has metadata.
|
||||
self.worker._remote_agents["old"][(0, 1)] = "old-other-rank"
|
||||
self.nixl.invalid_remote_agents.clear()
|
||||
if missing is True:
|
||||
self.nixl.invalid_remote_agents.add("old-other-rank")
|
||||
elif isinstance(missing, Exception):
|
||||
self.transport.check_remote_metadata.side_effect = missing
|
||||
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == result.failed_recving == {"old-req"}
|
||||
assert self.worker.get_block_ids_with_load_errors() == {1}
|
||||
assert ("old" not in self.worker._remote_agents) is (missing is True)
|
||||
assert self.transport.release_dlist_handle.call_count == (missing is True)
|
||||
|
||||
@pytest.mark.parametrize("handshake_fails", [False, True])
|
||||
def test_recovery_rehandshakes_same_id_with_fresh_descriptors(
|
||||
self, handshake_fails
|
||||
):
|
||||
old_handle = self.worker.dst_xfer_side_handles["old"][0]
|
||||
self._fail_old_peer()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.release_dlist_handle.assert_called_once_with(old_handle)
|
||||
assert not self.worker._engine_by_address
|
||||
with pytest.raises(KeyError):
|
||||
self.worker.transfer_topo.get_engine_info("old")
|
||||
self.transport.make_prepped_xfer.reset_mock()
|
||||
|
||||
self.worker.start_load_kv(self._request("old", req_id="next"))
|
||||
self.transport.make_prepped_xfer.assert_not_called()
|
||||
if handshake_fails:
|
||||
self.worker._handshake_futures["old"].set_exception(
|
||||
RuntimeError("handshake failed")
|
||||
)
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.failed_recving == result.finished_recving == {"next"}
|
||||
assert "old" not in self.worker._remote_agents
|
||||
assert self.worker._ready_requests.empty()
|
||||
return
|
||||
|
||||
self._connect("old")
|
||||
fresh_handle = self.worker.dst_xfer_side_handles["old"][0]
|
||||
assert fresh_handle != old_handle
|
||||
assert self.worker._engine_by_address == {("localhost", 1234): "old"}
|
||||
self.transport.check_xfer_state.return_value = "DONE"
|
||||
self.worker.start_load_kv(NixlConnectorMetadata())
|
||||
assert self.transport.make_prepped_xfer.call_args.args[3] == fresh_handle
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == {"next"} and not result.failed_recving
|
||||
self.transport.release_dlist_handle.assert_called_once_with(old_handle)
|
||||
|
||||
def test_recovery_drains_reads_and_rejects_new_ones_without_harming_other_peers(
|
||||
self,
|
||||
):
|
||||
self._connect("healthy", host="other-host")
|
||||
self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
|
||||
self.worker._recving_metadata.update(self._request("healthy").reqs_to_recv)
|
||||
self.worker._recving_transfers.update(
|
||||
{"old-req": [10, 11], "healthy-req": [12]}
|
||||
)
|
||||
self.nixl.invalid_remote_agents.add("old")
|
||||
self.transport.check_xfer_state.side_effect = (
|
||||
lambda h: "ERR" if h == 10 else "PROC"
|
||||
)
|
||||
old_handle = self.worker.dst_xfer_side_handles["old"][0]
|
||||
|
||||
result = self.worker.get_transfer_results()
|
||||
assert not result.finished_recving and not result.failed_recving
|
||||
self.transport.release_dlist_handle.assert_not_called()
|
||||
self.worker.start_load_kv(self._request("old", req_id="blocked"))
|
||||
self.transport.make_prepped_xfer.assert_not_called()
|
||||
|
||||
self.transport.check_xfer_state.side_effect = (
|
||||
lambda h: "DONE" if h == 11 else "PROC"
|
||||
)
|
||||
result = self.worker.get_transfer_results()
|
||||
assert (
|
||||
result.finished_recving == result.failed_recving == {"old-req", "blocked"}
|
||||
)
|
||||
assert set(self.worker._remote_agents) == {"healthy"}
|
||||
assert self.worker._recving_transfers == {"healthy-req": [12]}
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
calls = self.transport.mock_calls
|
||||
assert calls.index(call.release_xfer_handle(11)) < calls.index(
|
||||
call.release_dlist_handle(old_handle)
|
||||
)
|
||||
|
||||
def test_repeated_invalidation_backs_off_before_rehandshake(self):
|
||||
# A peer that is invalidated again soon after recovery (eg a persistent
|
||||
# RDMA fault) must not re-handshake on every request.
|
||||
self._fail_old_peer()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
self._connect("old")
|
||||
self._fail_old_peer()
|
||||
assert self.worker.get_transfer_results().failed_recving == {"old-req"}
|
||||
assert "old" in self.worker._remote_agents
|
||||
|
||||
self.transport.make_prepped_xfer.reset_mock()
|
||||
self.worker.start_load_kv(self._request("old", req_id="next"))
|
||||
assert self.worker.get_transfer_results().failed_recving == {"next"}
|
||||
self.transport.make_prepped_xfer.assert_not_called()
|
||||
assert "old" not in self.worker._handshake_futures
|
||||
self.transport.remove_remote_agent.assert_called_once()
|
||||
|
||||
backoff, release_at = self.worker._invalid_engine_backoff["old"]
|
||||
assert backoff == 1.0
|
||||
self.worker._invalid_engine_backoff["old"] = (backoff, release_at - backoff)
|
||||
self.worker.get_transfer_results()
|
||||
assert self.transport.remove_remote_agent.call_count == 2
|
||||
assert "old" not in self.worker._remote_agents
|
||||
|
||||
def test_recovery_waits_until_failed_handle_can_be_released(self):
|
||||
self._fail_old_peer()
|
||||
with patch.object(
|
||||
self.transport, "release_xfer_handle", side_effect=RuntimeError("in use")
|
||||
):
|
||||
result = self.worker.get_transfer_results()
|
||||
assert not result.finished_recving and not result.failed_recving
|
||||
assert self.worker._recving_transfers["old-req"]
|
||||
self.transport.release_dlist_handle.assert_not_called()
|
||||
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == result.failed_recving == {"old-req"}
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
@pytest.mark.parametrize("expired", [False, True])
|
||||
def test_success_and_expiry_do_not_probe_metadata(self, expired):
|
||||
metadata = self._request("old")
|
||||
if expired:
|
||||
metadata.reqs_to_recv["old-req"].remote.blocks_expiry_time = 0.0
|
||||
self.worker._bidirectional_kv_xfer_enabled = True
|
||||
with (
|
||||
patch.object(self.worker.xfer_stats, "record_failed_transfer") as failure,
|
||||
patch.object(self.worker.xfer_stats, "record_kv_expired_req") as expiry,
|
||||
):
|
||||
self.worker.start_load_kv(metadata)
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == {"old-req"}
|
||||
assert result.failed_recving == ({"old-req"} if expired else set())
|
||||
assert expiry.call_count == int(expired)
|
||||
failure.assert_not_called()
|
||||
self.transport.check_remote_metadata.assert_not_called()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize("awaiting_kvs", [False, True])
|
||||
def test_invalid_peer_empty_recv_is_reported_only_when_awaited(self, awaiting_kvs):
|
||||
self._fail_old_peer()
|
||||
self.worker._recving_transfers["old-req"].append(11)
|
||||
self.transport.check_xfer_state.side_effect = (
|
||||
lambda h: "PROC" if h == 11 else "ERR"
|
||||
)
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.make_prepped_xfer.reset_mock()
|
||||
|
||||
self.worker.start_load_kv(
|
||||
self._request("old", blocks=(), awaiting_kvs=awaiting_kvs, req_id="empty")
|
||||
)
|
||||
result = self.worker.get_transfer_results()
|
||||
expected = {"empty"} if awaiting_kvs else set()
|
||||
assert result.finished_recving == result.failed_recving == expected
|
||||
self.transport.make_prepped_xfer.assert_not_called()
|
||||
self.transport.send_notif.assert_not_called()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
def test_eviction_clears_pending_recovery(self):
|
||||
self.worker._failed_remote_engines.add("old")
|
||||
self.worker._invalid_remote_engines.add("old")
|
||||
self.worker._engine_ttl = 1
|
||||
self.worker._engine_last_active["old"] = time.perf_counter() - 10
|
||||
self.worker._evict_stale_engines()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
assert not self.worker._failed_remote_engines
|
||||
assert not self.worker._invalid_remote_engines
|
||||
|
||||
|
||||
def test_transfer_topology_unregister():
|
||||
"""TransferTopology.unregister_remote_engine removes the engine."""
|
||||
|
||||
@@ -60,6 +60,8 @@ def region_pull_worker():
|
||||
worker._transfer_layer_group_ids = ()
|
||||
worker._bidirectional_kv_xfer_enabled = False
|
||||
worker._recving_transfers = {}
|
||||
worker._failed_remote_engines = set()
|
||||
worker._invalid_remote_engines = set()
|
||||
worker.use_mla, worker._has_mamba = True, False
|
||||
worker.dcp_size = 1
|
||||
worker.dcp_rank = 0
|
||||
@@ -690,6 +692,8 @@ def test_read_blocks_for_req_expands_remote_ids(
|
||||
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker._physical_blocks_per_logical_kv_block = local_physical_per_logical
|
||||
worker._invalid_remote_engines = set()
|
||||
worker._remote_agents = {"remote-engine": {}}
|
||||
worker._engine_last_active = {}
|
||||
worker._recving_transfers = {}
|
||||
worker._bidirectional_kv_xfer_enabled = False
|
||||
|
||||
@@ -379,6 +379,8 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w.pp_size = 1
|
||||
w.engine_id = "test-decode-engine"
|
||||
w._remote_agents = {}
|
||||
w._failed_remote_engines = set()
|
||||
w._invalid_remote_engines = set()
|
||||
w._handshake_lock = threading.RLock()
|
||||
w._physical_blocks_per_logical_kv_block = 1
|
||||
w._uses_region_group_mapping = False
|
||||
|
||||
Reference in New Issue
Block a user