[Bugfix][NIXL] Recover locally invalidated pull peer metadata (#55471)
Signed-off-by: xijiade.aihemaiti <3146335281@qq.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yannik Hinteregger <37209495+YannikHinteregger@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Nick Hill
Yannik Hinteregger
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9bf4e51030
commit
9465e1c783
@@ -14,7 +14,7 @@ from collections import defaultdict
|
||||
from concurrent.futures import Future
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
@@ -144,6 +144,7 @@ class FakeNixlWrapper:
|
||||
def __init__(self, agent_name: str, *args, **kwargs):
|
||||
self._cycles_before_xfer_done = 0
|
||||
self._check_xfer_state_cycles: defaultdict[int, int] = defaultdict(lambda: 0)
|
||||
self.invalid_remote_agents: set[str] = set()
|
||||
|
||||
def get_reg_descs(self, caches_data, memory_type: str) -> list:
|
||||
return [str(uuid.uuid4()) for _ in caches_data]
|
||||
@@ -164,6 +165,7 @@ class FakeNixlWrapper:
|
||||
return self.AGENT_METADATA
|
||||
|
||||
def add_remote_agent(self, agent_metadata: bytes) -> str:
|
||||
self.invalid_remote_agents.discard(self.REMOTE_AGENT_NAME)
|
||||
return self.REMOTE_AGENT_NAME
|
||||
|
||||
def get_new_notifs(self) -> dict[str, list[bytes]]:
|
||||
@@ -185,6 +187,9 @@ class FakeNixlWrapper:
|
||||
def remove_remote_agent(self, agent: str) -> None:
|
||||
pass
|
||||
|
||||
def check_remote_metadata(self, agent: str) -> bool:
|
||||
return agent not in self.invalid_remote_agents
|
||||
|
||||
def send_notif(self, agent_name: str, notif_msg: bytes) -> None:
|
||||
pass
|
||||
|
||||
@@ -2127,6 +2132,8 @@ def recv_worker():
|
||||
worker._handshake_futures = {}
|
||||
worker._remote_agents = {}
|
||||
worker._engine_by_address = {}
|
||||
worker._failed_remote_engines = set()
|
||||
worker._invalid_remote_engines = set()
|
||||
worker._replicated_pcp_done_sending = set()
|
||||
worker._invalid_block_ids = queue.Queue()
|
||||
worker._pending_recv_notifs = {}
|
||||
@@ -2809,9 +2816,9 @@ class TestPeerReplacement:
|
||||
self.nixl.REMOTE_AGENT_NAME = engine_id
|
||||
future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id))
|
||||
|
||||
def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True):
|
||||
def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True, req_id=None):
|
||||
metadata = NixlConnectorMetadata()
|
||||
metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta(
|
||||
metadata.reqs_to_recv[req_id or f"{engine_id}-req"] = ReqMeta(
|
||||
local_block_ids=(blocks,),
|
||||
local_physical_block_ids=(blocks,),
|
||||
tp_size=1,
|
||||
@@ -2822,6 +2829,11 @@ class TestPeerReplacement:
|
||||
)
|
||||
return metadata
|
||||
|
||||
def _fail_old_peer(self):
|
||||
self.worker.start_load_kv(self._request("old"))
|
||||
self.nixl.invalid_remote_agents.add("old")
|
||||
self.transport.check_xfer_state.return_value = "ERR"
|
||||
|
||||
def test_cleans_old_peer_after_confirmed_handshake(self):
|
||||
self._connect("healthy", host="other-host")
|
||||
old_handle = self.worker.dst_xfer_side_handles["old"][0]
|
||||
@@ -2871,7 +2883,12 @@ class TestPeerReplacement:
|
||||
],
|
||||
ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"],
|
||||
)
|
||||
def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed):
|
||||
@pytest.mark.parametrize("invalidated", [False, True])
|
||||
def test_waits_for_reads_before_cleanup_and_failure_reporting(
|
||||
self, states, failed, invalidated
|
||||
):
|
||||
if invalidated:
|
||||
self.nixl.invalid_remote_agents.add("old")
|
||||
self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
|
||||
self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1]
|
||||
self.transport.check_xfer_state.side_effect = states
|
||||
@@ -2890,6 +2907,7 @@ class TestPeerReplacement:
|
||||
)
|
||||
assert self.transport.release_xfer_handle.call_count == (2 if failed else 1)
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
assert not self.worker._invalid_remote_engines
|
||||
|
||||
def test_waits_for_queued_notification(self):
|
||||
metadata = self._request("old", blocks=())
|
||||
@@ -2910,22 +2928,31 @@ class TestPeerReplacement:
|
||||
assert result.failed_recving == set()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
def test_waits_for_other_handshakes(self):
|
||||
@pytest.mark.parametrize("invalidated", [False, True])
|
||||
def test_waits_for_other_handshakes(self, invalidated):
|
||||
if invalidated:
|
||||
self._fail_old_peer()
|
||||
else:
|
||||
self._connect()
|
||||
self.worker._ensure_handshake("other", "other-host", 1234, 1)
|
||||
self._connect()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
self.transport.check_remote_metadata.assert_not_called()
|
||||
|
||||
self._connect("other", host="other-host")
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
def test_rehandshakes_queued_request_for_released_engine(self):
|
||||
@pytest.mark.parametrize("invalidated", [False, True])
|
||||
def test_rehandshakes_queued_request_for_released_engine(self, invalidated):
|
||||
# A request queued after its engine's handshake, but drained only after
|
||||
# that engine was replaced and released, must not read from it.
|
||||
if invalidated:
|
||||
self._fail_old_peer()
|
||||
metadata = self._request("old", blocks=(), awaiting_kvs=False)
|
||||
self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"]))
|
||||
self._connect()
|
||||
if not invalidated:
|
||||
self._connect()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
@@ -2933,6 +2960,185 @@ class TestPeerReplacement:
|
||||
assert "old" in self.worker._handshake_futures
|
||||
self.transport.send_notif.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize("missing", [False, True, RuntimeError("probe failed")])
|
||||
def test_recovery_requires_confirmed_missing_metadata(self, missing):
|
||||
self._fail_old_peer()
|
||||
# A missing rank suffices even if another rank still has metadata.
|
||||
self.worker._remote_agents["old"][(0, 1)] = "old-other-rank"
|
||||
self.nixl.invalid_remote_agents.clear()
|
||||
if missing is True:
|
||||
self.nixl.invalid_remote_agents.add("old-other-rank")
|
||||
elif isinstance(missing, Exception):
|
||||
self.transport.check_remote_metadata.side_effect = missing
|
||||
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == result.failed_recving == {"old-req"}
|
||||
assert self.worker.get_block_ids_with_load_errors() == {1}
|
||||
assert ("old" not in self.worker._remote_agents) is (missing is True)
|
||||
assert self.transport.release_dlist_handle.call_count == (missing is True)
|
||||
|
||||
@pytest.mark.parametrize("handshake_fails", [False, True])
|
||||
def test_recovery_rehandshakes_same_id_with_fresh_descriptors(
|
||||
self, handshake_fails
|
||||
):
|
||||
old_handle = self.worker.dst_xfer_side_handles["old"][0]
|
||||
self._fail_old_peer()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.release_dlist_handle.assert_called_once_with(old_handle)
|
||||
assert not self.worker._engine_by_address
|
||||
with pytest.raises(KeyError):
|
||||
self.worker.transfer_topo.get_engine_info("old")
|
||||
self.transport.make_prepped_xfer.reset_mock()
|
||||
|
||||
self.worker.start_load_kv(self._request("old", req_id="next"))
|
||||
self.transport.make_prepped_xfer.assert_not_called()
|
||||
if handshake_fails:
|
||||
self.worker._handshake_futures["old"].set_exception(
|
||||
RuntimeError("handshake failed")
|
||||
)
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.failed_recving == result.finished_recving == {"next"}
|
||||
assert "old" not in self.worker._remote_agents
|
||||
assert self.worker._ready_requests.empty()
|
||||
return
|
||||
|
||||
self._connect("old")
|
||||
fresh_handle = self.worker.dst_xfer_side_handles["old"][0]
|
||||
assert fresh_handle != old_handle
|
||||
assert self.worker._engine_by_address == {("localhost", 1234): "old"}
|
||||
self.transport.check_xfer_state.return_value = "DONE"
|
||||
self.worker.start_load_kv(NixlConnectorMetadata())
|
||||
assert self.transport.make_prepped_xfer.call_args.args[3] == fresh_handle
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == {"next"} and not result.failed_recving
|
||||
self.transport.release_dlist_handle.assert_called_once_with(old_handle)
|
||||
|
||||
def test_recovery_drains_reads_and_rejects_new_ones_without_harming_other_peers(
|
||||
self,
|
||||
):
|
||||
self._connect("healthy", host="other-host")
|
||||
self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
|
||||
self.worker._recving_metadata.update(self._request("healthy").reqs_to_recv)
|
||||
self.worker._recving_transfers.update(
|
||||
{"old-req": [10, 11], "healthy-req": [12]}
|
||||
)
|
||||
self.nixl.invalid_remote_agents.add("old")
|
||||
self.transport.check_xfer_state.side_effect = (
|
||||
lambda h: "ERR" if h == 10 else "PROC"
|
||||
)
|
||||
old_handle = self.worker.dst_xfer_side_handles["old"][0]
|
||||
|
||||
result = self.worker.get_transfer_results()
|
||||
assert not result.finished_recving and not result.failed_recving
|
||||
self.transport.release_dlist_handle.assert_not_called()
|
||||
self.worker.start_load_kv(self._request("old", req_id="blocked"))
|
||||
self.transport.make_prepped_xfer.assert_not_called()
|
||||
|
||||
self.transport.check_xfer_state.side_effect = (
|
||||
lambda h: "DONE" if h == 11 else "PROC"
|
||||
)
|
||||
result = self.worker.get_transfer_results()
|
||||
assert (
|
||||
result.finished_recving == result.failed_recving == {"old-req", "blocked"}
|
||||
)
|
||||
assert set(self.worker._remote_agents) == {"healthy"}
|
||||
assert self.worker._recving_transfers == {"healthy-req": [12]}
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
calls = self.transport.mock_calls
|
||||
assert calls.index(call.release_xfer_handle(11)) < calls.index(
|
||||
call.release_dlist_handle(old_handle)
|
||||
)
|
||||
|
||||
def test_repeated_invalidation_backs_off_before_rehandshake(self):
|
||||
# A peer that is invalidated again soon after recovery (eg a persistent
|
||||
# RDMA fault) must not re-handshake on every request.
|
||||
self._fail_old_peer()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
self._connect("old")
|
||||
self._fail_old_peer()
|
||||
assert self.worker.get_transfer_results().failed_recving == {"old-req"}
|
||||
assert "old" in self.worker._remote_agents
|
||||
|
||||
self.transport.make_prepped_xfer.reset_mock()
|
||||
self.worker.start_load_kv(self._request("old", req_id="next"))
|
||||
assert self.worker.get_transfer_results().failed_recving == {"next"}
|
||||
self.transport.make_prepped_xfer.assert_not_called()
|
||||
assert "old" not in self.worker._handshake_futures
|
||||
self.transport.remove_remote_agent.assert_called_once()
|
||||
|
||||
backoff, release_at = self.worker._invalid_engine_backoff["old"]
|
||||
assert backoff == 1.0
|
||||
self.worker._invalid_engine_backoff["old"] = (backoff, release_at - backoff)
|
||||
self.worker.get_transfer_results()
|
||||
assert self.transport.remove_remote_agent.call_count == 2
|
||||
assert "old" not in self.worker._remote_agents
|
||||
|
||||
def test_recovery_waits_until_failed_handle_can_be_released(self):
|
||||
self._fail_old_peer()
|
||||
with patch.object(
|
||||
self.transport, "release_xfer_handle", side_effect=RuntimeError("in use")
|
||||
):
|
||||
result = self.worker.get_transfer_results()
|
||||
assert not result.finished_recving and not result.failed_recving
|
||||
assert self.worker._recving_transfers["old-req"]
|
||||
self.transport.release_dlist_handle.assert_not_called()
|
||||
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == result.failed_recving == {"old-req"}
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
|
||||
@pytest.mark.parametrize("expired", [False, True])
|
||||
def test_success_and_expiry_do_not_probe_metadata(self, expired):
|
||||
metadata = self._request("old")
|
||||
if expired:
|
||||
metadata.reqs_to_recv["old-req"].remote.blocks_expiry_time = 0.0
|
||||
self.worker._bidirectional_kv_xfer_enabled = True
|
||||
with (
|
||||
patch.object(self.worker.xfer_stats, "record_failed_transfer") as failure,
|
||||
patch.object(self.worker.xfer_stats, "record_kv_expired_req") as expiry,
|
||||
):
|
||||
self.worker.start_load_kv(metadata)
|
||||
result = self.worker.get_transfer_results()
|
||||
assert result.finished_recving == {"old-req"}
|
||||
assert result.failed_recving == ({"old-req"} if expired else set())
|
||||
assert expiry.call_count == int(expired)
|
||||
failure.assert_not_called()
|
||||
self.transport.check_remote_metadata.assert_not_called()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize("awaiting_kvs", [False, True])
|
||||
def test_invalid_peer_empty_recv_is_reported_only_when_awaited(self, awaiting_kvs):
|
||||
self._fail_old_peer()
|
||||
self.worker._recving_transfers["old-req"].append(11)
|
||||
self.transport.check_xfer_state.side_effect = (
|
||||
lambda h: "PROC" if h == 11 else "ERR"
|
||||
)
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.make_prepped_xfer.reset_mock()
|
||||
|
||||
self.worker.start_load_kv(
|
||||
self._request("old", blocks=(), awaiting_kvs=awaiting_kvs, req_id="empty")
|
||||
)
|
||||
result = self.worker.get_transfer_results()
|
||||
expected = {"empty"} if awaiting_kvs else set()
|
||||
assert result.finished_recving == result.failed_recving == expected
|
||||
self.transport.make_prepped_xfer.assert_not_called()
|
||||
self.transport.send_notif.assert_not_called()
|
||||
self.transport.remove_remote_agent.assert_not_called()
|
||||
|
||||
def test_eviction_clears_pending_recovery(self):
|
||||
self.worker._failed_remote_engines.add("old")
|
||||
self.worker._invalid_remote_engines.add("old")
|
||||
self.worker._engine_ttl = 1
|
||||
self.worker._engine_last_active["old"] = time.perf_counter() - 10
|
||||
self.worker._evict_stale_engines()
|
||||
self.worker.get_transfer_results()
|
||||
self.transport.remove_remote_agent.assert_called_once_with("old")
|
||||
assert not self.worker._failed_remote_engines
|
||||
assert not self.worker._invalid_remote_engines
|
||||
|
||||
|
||||
def test_transfer_topology_unregister():
|
||||
"""TransferTopology.unregister_remote_engine removes the engine."""
|
||||
|
||||
@@ -60,6 +60,8 @@ def region_pull_worker():
|
||||
worker._transfer_layer_group_ids = ()
|
||||
worker._bidirectional_kv_xfer_enabled = False
|
||||
worker._recving_transfers = {}
|
||||
worker._failed_remote_engines = set()
|
||||
worker._invalid_remote_engines = set()
|
||||
worker.use_mla, worker._has_mamba = True, False
|
||||
worker.dcp_size = 1
|
||||
worker.dcp_rank = 0
|
||||
@@ -690,6 +692,8 @@ def test_read_blocks_for_req_expands_remote_ids(
|
||||
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker._physical_blocks_per_logical_kv_block = local_physical_per_logical
|
||||
worker._invalid_remote_engines = set()
|
||||
worker._remote_agents = {"remote-engine": {}}
|
||||
worker._engine_last_active = {}
|
||||
worker._recving_transfers = {}
|
||||
worker._bidirectional_kv_xfer_enabled = False
|
||||
|
||||
@@ -379,6 +379,8 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w.pp_size = 1
|
||||
w.engine_id = "test-decode-engine"
|
||||
w._remote_agents = {}
|
||||
w._failed_remote_engines = set()
|
||||
w._invalid_remote_engines = set()
|
||||
w._handshake_lock = threading.RLock()
|
||||
w._physical_blocks_per_logical_kv_block = 1
|
||||
w._uses_region_group_mapping = False
|
||||
|
||||
@@ -101,6 +101,11 @@ logger = init_logger(__name__)
|
||||
|
||||
_SHARED_REGION_GROUP_ID = -1
|
||||
|
||||
# Re-handshake delay bounds for a pull peer whose local metadata keeps getting
|
||||
# invalidated, eg due to a persistent RDMA fault.
|
||||
_INVALID_PEER_MIN_BACKOFF_S = 1.0
|
||||
_INVALID_PEER_MAX_BACKOFF_S = 60.0
|
||||
|
||||
|
||||
def _region_sort_key(layer_name: str) -> tuple[tuple[int, int | str], ...]:
|
||||
"""Sort transfer regions in model-layer order, then by cache name."""
|
||||
@@ -685,6 +690,12 @@ class NixlBaseConnectorWorker:
|
||||
self._remote_agents: dict[EngineId, dict[tuple[int, int], str]] = defaultdict(
|
||||
dict
|
||||
)
|
||||
# Failed reads need a local metadata probe; invalid peers wait for drain.
|
||||
self._failed_remote_engines: set[EngineId] = set()
|
||||
self._invalid_remote_engines: set[EngineId] = set()
|
||||
# Invalid engine -> (backoff, earliest release time). Kept across release
|
||||
# so that repeat invalidations back off.
|
||||
self._invalid_engine_backoff: dict[EngineId, tuple[float, float]] = {}
|
||||
# Map of engine_id -> clock offset.
|
||||
self._engine_clock_offset: dict[EngineId, float] = {}
|
||||
|
||||
@@ -3029,7 +3040,7 @@ class NixlBaseConnectorWorker:
|
||||
# Handle timeout to avoid stranding blocks on remote.
|
||||
self._reap_expired_send_leases(done_sending)
|
||||
|
||||
self._cleanup_replaced_remote_engines()
|
||||
self._cleanup_remote_engines()
|
||||
return KVConnectorTransferResults(
|
||||
finished_sending=done_sending,
|
||||
finished_recving=done_recving,
|
||||
@@ -3195,6 +3206,12 @@ class NixlBaseConnectorWorker:
|
||||
"""
|
||||
if record_failed_transfer:
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
if (
|
||||
self._TRANSFER_MODE == "pull"
|
||||
and (meta := self._recving_metadata.get(req_id)) is not None
|
||||
and meta.remote is not None
|
||||
):
|
||||
self._failed_remote_engines.add(meta.remote.engine_id)
|
||||
if failed_req_ids is not None:
|
||||
failed_req_ids.add(req_id)
|
||||
return handle is None or self._try_release_xfer_handle(req_id, handle)
|
||||
@@ -3642,15 +3659,38 @@ class NixlBaseConnectorWorker:
|
||||
engine_id,
|
||||
)
|
||||
|
||||
def _cleanup_replaced_remote_engines(self) -> None:
|
||||
"""Release replaced pull peers once requests and handshakes have drained."""
|
||||
def _cleanup_remote_engines(self) -> None:
|
||||
"""Release replaced or locally invalid pull peers after their reads drain."""
|
||||
# TODO: Also handle push mode, which handshakes in both directions.
|
||||
if self._TRANSFER_MODE != "pull":
|
||||
return
|
||||
# A background handshake may be loading native metadata with the GIL
|
||||
# released. Defer both probing and cleanup until it has finished.
|
||||
with self._handshake_lock:
|
||||
if self._handshake_futures:
|
||||
return
|
||||
replaced = self._remote_agents.keys() - self._engine_by_address.values()
|
||||
if not replaced:
|
||||
now = time.perf_counter()
|
||||
for engine_id in self._failed_remote_engines - self._invalid_remote_engines:
|
||||
agents = self._remote_agents.get(engine_id, {}).values()
|
||||
try:
|
||||
# No descriptors: query local existence, not peer health.
|
||||
if not all(map(self.nixl_wrapper.check_remote_metadata, agents)):
|
||||
self._mark_remote_engine_invalid(engine_id, now)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Could not check local NIXL metadata for engine %s.",
|
||||
engine_id,
|
||||
exc_info=True,
|
||||
)
|
||||
self._failed_remote_engines.clear()
|
||||
# Invalid peers keep rejecting new reads until their backoff has elapsed.
|
||||
invalid = {
|
||||
engine_id
|
||||
for engine_id in self._invalid_remote_engines
|
||||
if self._invalid_engine_backoff.get(engine_id, (0.0, 0.0))[1] <= now
|
||||
}
|
||||
if not replaced and not invalid:
|
||||
return
|
||||
# _recving_metadata is only accessed from this (main) thread.
|
||||
busy = {
|
||||
@@ -3658,9 +3698,42 @@ class NixlBaseConnectorWorker:
|
||||
for meta in self._recving_metadata.values()
|
||||
if meta.remote is not None
|
||||
}
|
||||
for engine_id in replaced - busy:
|
||||
# Replaced peers can still serve queued requests. Invalid peers reject
|
||||
# new reads, but must retain descriptors until outstanding handles end.
|
||||
invalid -= self._engines_with_inflight_transfers()
|
||||
for engine_id in (replaced - busy) | invalid:
|
||||
self._cleanup_remote_engine(engine_id, log_eviction=False)
|
||||
logger.info("Released NIXL state for replaced remote engine %s.", engine_id)
|
||||
logger.info(
|
||||
"Released NIXL state for replaced or locally invalid remote engine %s.",
|
||||
engine_id,
|
||||
)
|
||||
|
||||
def _mark_remote_engine_invalid(self, engine_id: EngineId, now: float) -> None:
|
||||
"""Schedule release of an engine whose local NIXL metadata is gone.
|
||||
|
||||
The first invalidation is released as soon as reads drain. Repeats
|
||||
before the peer has stayed healthy for the max backoff double the
|
||||
delay, during which reads fail fast instead of re-handshaking.
|
||||
"""
|
||||
# Forget engines that have stayed healthy long enough.
|
||||
for eid, (_, release_at) in list(self._invalid_engine_backoff.items()):
|
||||
if now - release_at > _INVALID_PEER_MAX_BACKOFF_S:
|
||||
del self._invalid_engine_backoff[eid]
|
||||
prev = self._invalid_engine_backoff.get(engine_id)
|
||||
backoff = 0.0
|
||||
if prev is not None:
|
||||
backoff = min(
|
||||
max(2 * prev[0], _INVALID_PEER_MIN_BACKOFF_S),
|
||||
_INVALID_PEER_MAX_BACKOFF_S,
|
||||
)
|
||||
self._invalid_engine_backoff[engine_id] = (backoff, now + backoff)
|
||||
self._invalid_remote_engines.add(engine_id)
|
||||
logger.warning(
|
||||
"Local NIXL metadata for remote engine %s was invalidated; "
|
||||
"re-handshaking after reads drain%s.",
|
||||
engine_id,
|
||||
f" and a {backoff:.0f}s backoff" if backoff else "",
|
||||
)
|
||||
|
||||
def _cleanup_remote_engine(
|
||||
self, engine_id: EngineId, *, log_eviction: bool = True
|
||||
@@ -3668,8 +3741,8 @@ class NixlBaseConnectorWorker:
|
||||
"""Remove all state for a single remote engine.
|
||||
|
||||
Releases NIXL resources (dlist handles, remote agents) and clears
|
||||
all per-engine data structures. Used by TTL eviction, replaced peer
|
||||
cleanup and shutdown.
|
||||
all per-engine data structures. Used by TTL eviction, replaced/invalid
|
||||
peer cleanup and shutdown.
|
||||
"""
|
||||
assert engine_id in self._remote_agents
|
||||
|
||||
@@ -3685,6 +3758,9 @@ class NixlBaseConnectorWorker:
|
||||
for agent_name in agents.values():
|
||||
self.nixl_wrapper.remove_remote_agent(agent_name)
|
||||
|
||||
self._failed_remote_engines.discard(engine_id)
|
||||
self._invalid_remote_engines.discard(engine_id)
|
||||
|
||||
self.kv_caches_base_addr.pop(engine_id, None)
|
||||
self.dst_num_blocks.pop(engine_id, None)
|
||||
self.dst_region_num_blocks.pop(engine_id, None)
|
||||
|
||||
@@ -72,26 +72,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
# Aborted cleanup requests have already been removed from the scheduler.
|
||||
if meta.awaiting_kvs or any(meta.local_block_ids):
|
||||
self._recving_metadata[req_id] = meta
|
||||
if remote_engine_id not in self._remote_agents:
|
||||
# Initiate handshake with remote engine to exchange metadata.
|
||||
with self._handshake_lock:
|
||||
if remote_engine_id not in self._remote_agents:
|
||||
self._background_nixl_handshake(req_id, remote_engine_id, meta)
|
||||
continue
|
||||
|
||||
# Handshake already completed, start async read xfer.
|
||||
self._read_blocks_for_req(req_id, meta)
|
||||
|
||||
# Start transfers for requests whose handshakes have now finished.
|
||||
while not self._ready_requests.empty():
|
||||
req_id, meta = self._ready_requests.get_nowait()
|
||||
assert meta.remote is not None
|
||||
if meta.remote.engine_id not in self._remote_agents:
|
||||
# The engine was released after its handshake completed, so
|
||||
# handshake again. This fails if the engine is gone.
|
||||
self._background_nixl_handshake(req_id, meta.remote.engine_id, meta)
|
||||
continue
|
||||
self._read_blocks_for_req(req_id, meta)
|
||||
self._read_blocks_for_req(*self._ready_requests.get_nowait())
|
||||
|
||||
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
|
||||
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
|
||||
@@ -151,6 +136,15 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
|
||||
assert meta.remote is not None and self.transfer_topo is not None
|
||||
engine_id = meta.remote.engine_id
|
||||
if engine_id in self._invalid_remote_engines:
|
||||
self._handle_failed_transfer(
|
||||
req_id, None, self._recv_failures, record_failed_transfer=False
|
||||
)
|
||||
return
|
||||
if engine_id not in self._remote_agents:
|
||||
# Also re-handshakes queued requests that outlived invalid-peer cleanup.
|
||||
self._background_nixl_handshake(req_id, engine_id, meta)
|
||||
return
|
||||
# Update last activity from this remote. Mind that cleanup is done on main
|
||||
# thread (this one), so we don't race on this structure.
|
||||
self._engine_last_active[engine_id] = time.perf_counter()
|
||||
|
||||
Reference in New Issue
Block a user