diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 43a3c7030a..35ac7d887e 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -14,7 +14,7 @@ from collections import defaultdict from concurrent.futures import Future from types import SimpleNamespace from typing import Any, cast -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch import msgspec import numpy as np @@ -144,6 +144,7 @@ class FakeNixlWrapper: def __init__(self, agent_name: str, *args, **kwargs): self._cycles_before_xfer_done = 0 self._check_xfer_state_cycles: defaultdict[int, int] = defaultdict(lambda: 0) + self.invalid_remote_agents: set[str] = set() def get_reg_descs(self, caches_data, memory_type: str) -> list: return [str(uuid.uuid4()) for _ in caches_data] @@ -164,6 +165,7 @@ class FakeNixlWrapper: return self.AGENT_METADATA def add_remote_agent(self, agent_metadata: bytes) -> str: + self.invalid_remote_agents.discard(self.REMOTE_AGENT_NAME) return self.REMOTE_AGENT_NAME def get_new_notifs(self) -> dict[str, list[bytes]]: @@ -185,6 +187,9 @@ class FakeNixlWrapper: def remove_remote_agent(self, agent: str) -> None: pass + def check_remote_metadata(self, agent: str) -> bool: + return agent not in self.invalid_remote_agents + def send_notif(self, agent_name: str, notif_msg: bytes) -> None: pass @@ -2127,6 +2132,8 @@ def recv_worker(): worker._handshake_futures = {} worker._remote_agents = {} worker._engine_by_address = {} + worker._failed_remote_engines = set() + worker._invalid_remote_engines = set() worker._replicated_pcp_done_sending = set() worker._invalid_block_ids = queue.Queue() worker._pending_recv_notifs = {} @@ -2809,9 +2816,9 @@ class TestPeerReplacement: self.nixl.REMOTE_AGENT_NAME = engine_id future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id)) - def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True): + def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True, req_id=None): metadata = NixlConnectorMetadata() - metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta( + metadata.reqs_to_recv[req_id or f"{engine_id}-req"] = ReqMeta( local_block_ids=(blocks,), local_physical_block_ids=(blocks,), tp_size=1, @@ -2822,6 +2829,11 @@ class TestPeerReplacement: ) return metadata + def _fail_old_peer(self): + self.worker.start_load_kv(self._request("old")) + self.nixl.invalid_remote_agents.add("old") + self.transport.check_xfer_state.return_value = "ERR" + def test_cleans_old_peer_after_confirmed_handshake(self): self._connect("healthy", host="other-host") old_handle = self.worker.dst_xfer_side_handles["old"][0] @@ -2871,7 +2883,12 @@ class TestPeerReplacement: ], ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"], ) - def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed): + @pytest.mark.parametrize("invalidated", [False, True]) + def test_waits_for_reads_before_cleanup_and_failure_reporting( + self, states, failed, invalidated + ): + if invalidated: + self.nixl.invalid_remote_agents.add("old") self.worker._recving_metadata.update(self._request("old").reqs_to_recv) self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1] self.transport.check_xfer_state.side_effect = states @@ -2890,6 +2907,7 @@ class TestPeerReplacement: ) assert self.transport.release_xfer_handle.call_count == (2 if failed else 1) self.transport.remove_remote_agent.assert_called_once_with("old") + assert not self.worker._invalid_remote_engines def test_waits_for_queued_notification(self): metadata = self._request("old", blocks=()) @@ -2910,22 +2928,31 @@ class TestPeerReplacement: assert result.failed_recving == set() self.transport.remove_remote_agent.assert_called_once_with("old") - def test_waits_for_other_handshakes(self): + @pytest.mark.parametrize("invalidated", [False, True]) + def test_waits_for_other_handshakes(self, invalidated): + if invalidated: + self._fail_old_peer() + else: + self._connect() self.worker._ensure_handshake("other", "other-host", 1234, 1) - self._connect() self.worker.get_transfer_results() self.transport.remove_remote_agent.assert_not_called() + self.transport.check_remote_metadata.assert_not_called() self._connect("other", host="other-host") self.worker.get_transfer_results() self.transport.remove_remote_agent.assert_called_once_with("old") - def test_rehandshakes_queued_request_for_released_engine(self): + @pytest.mark.parametrize("invalidated", [False, True]) + def test_rehandshakes_queued_request_for_released_engine(self, invalidated): # A request queued after its engine's handshake, but drained only after # that engine was replaced and released, must not read from it. + if invalidated: + self._fail_old_peer() metadata = self._request("old", blocks=(), awaiting_kvs=False) self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"])) - self._connect() + if not invalidated: + self._connect() self.worker.get_transfer_results() self.transport.remove_remote_agent.assert_called_once_with("old") @@ -2933,6 +2960,185 @@ class TestPeerReplacement: assert "old" in self.worker._handshake_futures self.transport.send_notif.assert_not_called() + @pytest.mark.parametrize("missing", [False, True, RuntimeError("probe failed")]) + def test_recovery_requires_confirmed_missing_metadata(self, missing): + self._fail_old_peer() + # A missing rank suffices even if another rank still has metadata. + self.worker._remote_agents["old"][(0, 1)] = "old-other-rank" + self.nixl.invalid_remote_agents.clear() + if missing is True: + self.nixl.invalid_remote_agents.add("old-other-rank") + elif isinstance(missing, Exception): + self.transport.check_remote_metadata.side_effect = missing + + result = self.worker.get_transfer_results() + assert result.finished_recving == result.failed_recving == {"old-req"} + assert self.worker.get_block_ids_with_load_errors() == {1} + assert ("old" not in self.worker._remote_agents) is (missing is True) + assert self.transport.release_dlist_handle.call_count == (missing is True) + + @pytest.mark.parametrize("handshake_fails", [False, True]) + def test_recovery_rehandshakes_same_id_with_fresh_descriptors( + self, handshake_fails + ): + old_handle = self.worker.dst_xfer_side_handles["old"][0] + self._fail_old_peer() + self.worker.get_transfer_results() + self.transport.release_dlist_handle.assert_called_once_with(old_handle) + assert not self.worker._engine_by_address + with pytest.raises(KeyError): + self.worker.transfer_topo.get_engine_info("old") + self.transport.make_prepped_xfer.reset_mock() + + self.worker.start_load_kv(self._request("old", req_id="next")) + self.transport.make_prepped_xfer.assert_not_called() + if handshake_fails: + self.worker._handshake_futures["old"].set_exception( + RuntimeError("handshake failed") + ) + result = self.worker.get_transfer_results() + assert result.failed_recving == result.finished_recving == {"next"} + assert "old" not in self.worker._remote_agents + assert self.worker._ready_requests.empty() + return + + self._connect("old") + fresh_handle = self.worker.dst_xfer_side_handles["old"][0] + assert fresh_handle != old_handle + assert self.worker._engine_by_address == {("localhost", 1234): "old"} + self.transport.check_xfer_state.return_value = "DONE" + self.worker.start_load_kv(NixlConnectorMetadata()) + assert self.transport.make_prepped_xfer.call_args.args[3] == fresh_handle + result = self.worker.get_transfer_results() + assert result.finished_recving == {"next"} and not result.failed_recving + self.transport.release_dlist_handle.assert_called_once_with(old_handle) + + def test_recovery_drains_reads_and_rejects_new_ones_without_harming_other_peers( + self, + ): + self._connect("healthy", host="other-host") + self.worker._recving_metadata.update(self._request("old").reqs_to_recv) + self.worker._recving_metadata.update(self._request("healthy").reqs_to_recv) + self.worker._recving_transfers.update( + {"old-req": [10, 11], "healthy-req": [12]} + ) + self.nixl.invalid_remote_agents.add("old") + self.transport.check_xfer_state.side_effect = ( + lambda h: "ERR" if h == 10 else "PROC" + ) + old_handle = self.worker.dst_xfer_side_handles["old"][0] + + result = self.worker.get_transfer_results() + assert not result.finished_recving and not result.failed_recving + self.transport.release_dlist_handle.assert_not_called() + self.worker.start_load_kv(self._request("old", req_id="blocked")) + self.transport.make_prepped_xfer.assert_not_called() + + self.transport.check_xfer_state.side_effect = ( + lambda h: "DONE" if h == 11 else "PROC" + ) + result = self.worker.get_transfer_results() + assert ( + result.finished_recving == result.failed_recving == {"old-req", "blocked"} + ) + assert set(self.worker._remote_agents) == {"healthy"} + assert self.worker._recving_transfers == {"healthy-req": [12]} + self.transport.remove_remote_agent.assert_called_once_with("old") + calls = self.transport.mock_calls + assert calls.index(call.release_xfer_handle(11)) < calls.index( + call.release_dlist_handle(old_handle) + ) + + def test_repeated_invalidation_backs_off_before_rehandshake(self): + # A peer that is invalidated again soon after recovery (eg a persistent + # RDMA fault) must not re-handshake on every request. + self._fail_old_peer() + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_called_once_with("old") + + self._connect("old") + self._fail_old_peer() + assert self.worker.get_transfer_results().failed_recving == {"old-req"} + assert "old" in self.worker._remote_agents + + self.transport.make_prepped_xfer.reset_mock() + self.worker.start_load_kv(self._request("old", req_id="next")) + assert self.worker.get_transfer_results().failed_recving == {"next"} + self.transport.make_prepped_xfer.assert_not_called() + assert "old" not in self.worker._handshake_futures + self.transport.remove_remote_agent.assert_called_once() + + backoff, release_at = self.worker._invalid_engine_backoff["old"] + assert backoff == 1.0 + self.worker._invalid_engine_backoff["old"] = (backoff, release_at - backoff) + self.worker.get_transfer_results() + assert self.transport.remove_remote_agent.call_count == 2 + assert "old" not in self.worker._remote_agents + + def test_recovery_waits_until_failed_handle_can_be_released(self): + self._fail_old_peer() + with patch.object( + self.transport, "release_xfer_handle", side_effect=RuntimeError("in use") + ): + result = self.worker.get_transfer_results() + assert not result.finished_recving and not result.failed_recving + assert self.worker._recving_transfers["old-req"] + self.transport.release_dlist_handle.assert_not_called() + + result = self.worker.get_transfer_results() + assert result.finished_recving == result.failed_recving == {"old-req"} + self.transport.remove_remote_agent.assert_called_once_with("old") + + @pytest.mark.parametrize("expired", [False, True]) + def test_success_and_expiry_do_not_probe_metadata(self, expired): + metadata = self._request("old") + if expired: + metadata.reqs_to_recv["old-req"].remote.blocks_expiry_time = 0.0 + self.worker._bidirectional_kv_xfer_enabled = True + with ( + patch.object(self.worker.xfer_stats, "record_failed_transfer") as failure, + patch.object(self.worker.xfer_stats, "record_kv_expired_req") as expiry, + ): + self.worker.start_load_kv(metadata) + result = self.worker.get_transfer_results() + assert result.finished_recving == {"old-req"} + assert result.failed_recving == ({"old-req"} if expired else set()) + assert expiry.call_count == int(expired) + failure.assert_not_called() + self.transport.check_remote_metadata.assert_not_called() + self.transport.remove_remote_agent.assert_not_called() + + @pytest.mark.parametrize("awaiting_kvs", [False, True]) + def test_invalid_peer_empty_recv_is_reported_only_when_awaited(self, awaiting_kvs): + self._fail_old_peer() + self.worker._recving_transfers["old-req"].append(11) + self.transport.check_xfer_state.side_effect = ( + lambda h: "PROC" if h == 11 else "ERR" + ) + self.worker.get_transfer_results() + self.transport.make_prepped_xfer.reset_mock() + + self.worker.start_load_kv( + self._request("old", blocks=(), awaiting_kvs=awaiting_kvs, req_id="empty") + ) + result = self.worker.get_transfer_results() + expected = {"empty"} if awaiting_kvs else set() + assert result.finished_recving == result.failed_recving == expected + self.transport.make_prepped_xfer.assert_not_called() + self.transport.send_notif.assert_not_called() + self.transport.remove_remote_agent.assert_not_called() + + def test_eviction_clears_pending_recovery(self): + self.worker._failed_remote_engines.add("old") + self.worker._invalid_remote_engines.add("old") + self.worker._engine_ttl = 1 + self.worker._engine_last_active["old"] = time.perf_counter() - 10 + self.worker._evict_stale_engines() + self.worker.get_transfer_results() + self.transport.remove_remote_agent.assert_called_once_with("old") + assert not self.worker._failed_remote_engines + assert not self.worker._invalid_remote_engines + def test_transfer_topology_unregister(): """TransferTopology.unregister_remote_engine removes the engine.""" diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 2a1ca5a877..f965955721 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -60,6 +60,8 @@ def region_pull_worker(): worker._transfer_layer_group_ids = () worker._bidirectional_kv_xfer_enabled = False worker._recving_transfers = {} + worker._failed_remote_engines = set() + worker._invalid_remote_engines = set() worker.use_mla, worker._has_mamba = True, False worker.dcp_size = 1 worker.dcp_rank = 0 @@ -690,6 +692,8 @@ def test_read_blocks_for_req_expands_remote_ids( worker = object.__new__(NixlConnectorWorker) worker._physical_blocks_per_logical_kv_block = local_physical_per_logical + worker._invalid_remote_engines = set() + worker._remote_agents = {"remote-engine": {}} worker._engine_last_active = {} worker._recving_transfers = {} worker._bidirectional_kv_xfer_enabled = False diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index 47779c3611..a36c7e5b2a 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -379,6 +379,8 @@ class _StubWriterWorker(NixlPushConnectorWorker): w.pp_size = 1 w.engine_id = "test-decode-engine" w._remote_agents = {} + w._failed_remote_engines = set() + w._invalid_remote_engines = set() w._handshake_lock = threading.RLock() w._physical_blocks_per_logical_kv_block = 1 w._uses_region_group_mapping = False diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index dade5056ee..fa29d41a82 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -101,6 +101,11 @@ logger = init_logger(__name__) _SHARED_REGION_GROUP_ID = -1 +# Re-handshake delay bounds for a pull peer whose local metadata keeps getting +# invalidated, eg due to a persistent RDMA fault. +_INVALID_PEER_MIN_BACKOFF_S = 1.0 +_INVALID_PEER_MAX_BACKOFF_S = 60.0 + def _region_sort_key(layer_name: str) -> tuple[tuple[int, int | str], ...]: """Sort transfer regions in model-layer order, then by cache name.""" @@ -685,6 +690,12 @@ class NixlBaseConnectorWorker: self._remote_agents: dict[EngineId, dict[tuple[int, int], str]] = defaultdict( dict ) + # Failed reads need a local metadata probe; invalid peers wait for drain. + self._failed_remote_engines: set[EngineId] = set() + self._invalid_remote_engines: set[EngineId] = set() + # Invalid engine -> (backoff, earliest release time). Kept across release + # so that repeat invalidations back off. + self._invalid_engine_backoff: dict[EngineId, tuple[float, float]] = {} # Map of engine_id -> clock offset. self._engine_clock_offset: dict[EngineId, float] = {} @@ -3029,7 +3040,7 @@ class NixlBaseConnectorWorker: # Handle timeout to avoid stranding blocks on remote. self._reap_expired_send_leases(done_sending) - self._cleanup_replaced_remote_engines() + self._cleanup_remote_engines() return KVConnectorTransferResults( finished_sending=done_sending, finished_recving=done_recving, @@ -3195,6 +3206,12 @@ class NixlBaseConnectorWorker: """ if record_failed_transfer: self.xfer_stats.record_failed_transfer() + if ( + self._TRANSFER_MODE == "pull" + and (meta := self._recving_metadata.get(req_id)) is not None + and meta.remote is not None + ): + self._failed_remote_engines.add(meta.remote.engine_id) if failed_req_ids is not None: failed_req_ids.add(req_id) return handle is None or self._try_release_xfer_handle(req_id, handle) @@ -3642,15 +3659,38 @@ class NixlBaseConnectorWorker: engine_id, ) - def _cleanup_replaced_remote_engines(self) -> None: - """Release replaced pull peers once requests and handshakes have drained.""" + def _cleanup_remote_engines(self) -> None: + """Release replaced or locally invalid pull peers after their reads drain.""" + # TODO: Also handle push mode, which handshakes in both directions. if self._TRANSFER_MODE != "pull": return + # A background handshake may be loading native metadata with the GIL + # released. Defer both probing and cleanup until it has finished. with self._handshake_lock: if self._handshake_futures: return replaced = self._remote_agents.keys() - self._engine_by_address.values() - if not replaced: + now = time.perf_counter() + for engine_id in self._failed_remote_engines - self._invalid_remote_engines: + agents = self._remote_agents.get(engine_id, {}).values() + try: + # No descriptors: query local existence, not peer health. + if not all(map(self.nixl_wrapper.check_remote_metadata, agents)): + self._mark_remote_engine_invalid(engine_id, now) + except Exception: + logger.warning( + "Could not check local NIXL metadata for engine %s.", + engine_id, + exc_info=True, + ) + self._failed_remote_engines.clear() + # Invalid peers keep rejecting new reads until their backoff has elapsed. + invalid = { + engine_id + for engine_id in self._invalid_remote_engines + if self._invalid_engine_backoff.get(engine_id, (0.0, 0.0))[1] <= now + } + if not replaced and not invalid: return # _recving_metadata is only accessed from this (main) thread. busy = { @@ -3658,9 +3698,42 @@ class NixlBaseConnectorWorker: for meta in self._recving_metadata.values() if meta.remote is not None } - for engine_id in replaced - busy: + # Replaced peers can still serve queued requests. Invalid peers reject + # new reads, but must retain descriptors until outstanding handles end. + invalid -= self._engines_with_inflight_transfers() + for engine_id in (replaced - busy) | invalid: self._cleanup_remote_engine(engine_id, log_eviction=False) - logger.info("Released NIXL state for replaced remote engine %s.", engine_id) + logger.info( + "Released NIXL state for replaced or locally invalid remote engine %s.", + engine_id, + ) + + def _mark_remote_engine_invalid(self, engine_id: EngineId, now: float) -> None: + """Schedule release of an engine whose local NIXL metadata is gone. + + The first invalidation is released as soon as reads drain. Repeats + before the peer has stayed healthy for the max backoff double the + delay, during which reads fail fast instead of re-handshaking. + """ + # Forget engines that have stayed healthy long enough. + for eid, (_, release_at) in list(self._invalid_engine_backoff.items()): + if now - release_at > _INVALID_PEER_MAX_BACKOFF_S: + del self._invalid_engine_backoff[eid] + prev = self._invalid_engine_backoff.get(engine_id) + backoff = 0.0 + if prev is not None: + backoff = min( + max(2 * prev[0], _INVALID_PEER_MIN_BACKOFF_S), + _INVALID_PEER_MAX_BACKOFF_S, + ) + self._invalid_engine_backoff[engine_id] = (backoff, now + backoff) + self._invalid_remote_engines.add(engine_id) + logger.warning( + "Local NIXL metadata for remote engine %s was invalidated; " + "re-handshaking after reads drain%s.", + engine_id, + f" and a {backoff:.0f}s backoff" if backoff else "", + ) def _cleanup_remote_engine( self, engine_id: EngineId, *, log_eviction: bool = True @@ -3668,8 +3741,8 @@ class NixlBaseConnectorWorker: """Remove all state for a single remote engine. Releases NIXL resources (dlist handles, remote agents) and clears - all per-engine data structures. Used by TTL eviction, replaced peer - cleanup and shutdown. + all per-engine data structures. Used by TTL eviction, replaced/invalid + peer cleanup and shutdown. """ assert engine_id in self._remote_agents @@ -3685,6 +3758,9 @@ class NixlBaseConnectorWorker: for agent_name in agents.values(): self.nixl_wrapper.remove_remote_agent(agent_name) + self._failed_remote_engines.discard(engine_id) + self._invalid_remote_engines.discard(engine_id) + self.kv_caches_base_addr.pop(engine_id, None) self.dst_num_blocks.pop(engine_id, None) self.dst_region_num_blocks.pop(engine_id, None) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index 4a18632eb4..5bcfdb9560 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -72,26 +72,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): # Aborted cleanup requests have already been removed from the scheduler. if meta.awaiting_kvs or any(meta.local_block_ids): self._recving_metadata[req_id] = meta - if remote_engine_id not in self._remote_agents: - # Initiate handshake with remote engine to exchange metadata. - with self._handshake_lock: - if remote_engine_id not in self._remote_agents: - self._background_nixl_handshake(req_id, remote_engine_id, meta) - continue - - # Handshake already completed, start async read xfer. self._read_blocks_for_req(req_id, meta) # Start transfers for requests whose handshakes have now finished. while not self._ready_requests.empty(): - req_id, meta = self._ready_requests.get_nowait() - assert meta.remote is not None - if meta.remote.engine_id not in self._remote_agents: - # The engine was released after its handshake completed, so - # handshake again. This fails if the engine is gone. - self._background_nixl_handshake(req_id, meta.remote.engine_id, meta) - continue - self._read_blocks_for_req(req_id, meta) + self._read_blocks_for_req(*self._ready_requests.get_nowait()) if self.pcp_rank > 0 and not self.pcp_dcp_sharded: # Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank @@ -151,6 +136,15 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): def _read_blocks_for_req(self, req_id: str, meta: ReqMeta): assert meta.remote is not None and self.transfer_topo is not None engine_id = meta.remote.engine_id + if engine_id in self._invalid_remote_engines: + self._handle_failed_transfer( + req_id, None, self._recv_failures, record_failed_transfer=False + ) + return + if engine_id not in self._remote_agents: + # Also re-handshakes queued requests that outlived invalid-peer cleanup. + self._background_nixl_handshake(req_id, engine_id, meta) + return # Update last activity from this remote. Mind that cleanup is done on main # thread (this one), so we don't race on this structure. self._engine_last_active[engine_id] = time.perf_counter()