diff --git a/tests/v1/kv_offload/tiering/p2p/test_data_transport.py b/tests/v1/kv_offload/tiering/p2p/test_data_transport.py index 7219d691fd..e7da4438a1 100644 --- a/tests/v1/kv_offload/tiering/p2p/test_data_transport.py +++ b/tests/v1/kv_offload/tiering/p2p/test_data_transport.py @@ -11,6 +11,7 @@ from unittest.mock import MagicMock, patch import numpy as np import pytest +from vllm.v1.kv_offload.tiering.p2p.data import nixl as nixl_module from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport @@ -425,6 +426,71 @@ class TestNixlTransportWithMockedAgent: transport._agent.release_dlist_handle.assert_called() transport._agent.remove_remote_agent.assert_called() + def test_remove_peer_removes_agent_even_if_release_fails(self): + """A dlist release failure must not skip the agent removal. + + Teardown runs two independent NIXL operations. If + release_dlist_handle() raises, remove_remote_agent() must still be + attempted with the registered agent name; skipping it would leave + a stale NIXL registration behind after the peer is gone. + """ + transport = self._make_transport() + transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) + transport._agent.release_dlist_handle.side_effect = RuntimeError( + "simulated release failure" + ) + + with patch.object(nixl_module.logger, "warning") as mock_warning: + transport.remove_remote_peer("peer:1") + + # The release failure was logged rather than swallowed silently. + mock_warning.assert_called_once() + # Agent removal was still attempted with the registered name. + transport._agent.release_dlist_handle.assert_called_once() + transport._agent.remove_remote_agent.assert_called_once_with("nixl-peer-name") + assert "peer:1" not in transport._remote_dlists + + def test_superseded_registration_removes_agent_even_if_release_fails(self): + """A release failure while discarding must still remove the agent. + + _publish_peer discards handles built under a stale generation when + the peer was removed or re-registered mid-registration. Its + descriptor-list release and agent removal are independent too: if + release_dlist_handle() raises, remove_remote_agent() must still be + attempted for the superseded registration, otherwise its remote + agent stays registered forever. + """ + transport = self._make_transport() + building = threading.Event() + release = threading.Event() + + def blocking_prep(*args, **kwargs): + building.set() + release.wait(timeout=5) + return MagicMock(name="remote_dlist") + + transport._agent.prep_xfer_dlist.side_effect = blocking_prep + + future = transport.add_remote_peer_async("peer:1", b"meta", 0x1000, 8, 1024) + assert building.wait(timeout=5) + # Supersede the registration while its worker still builds handles. + transport.remove_remote_peer("peer:1") + transport._agent.release_dlist_handle.side_effect = RuntimeError( + "simulated release failure" + ) + + with patch.object(nixl_module.logger, "warning") as mock_warning: + release.set() + with pytest.raises(RuntimeError, match="superseded"): + future.result(timeout=5) + + # The release failure was logged rather than swallowed silently. + mock_warning.assert_called_once() + # The superseded registration's agent was still removed. + transport._agent.release_dlist_handle.assert_called_once() + transport._agent.remove_remote_agent.assert_called_once_with("nixl-peer-name") + assert "peer:1" not in transport._remote_dlists + def test_close_releases_everything(self): """Close releases all handles and clears state.""" transport = self._make_transport() diff --git a/vllm/v1/kv_offload/tiering/p2p/data/nixl.py b/vllm/v1/kv_offload/tiering/p2p/data/nixl.py index 8f32ef859a..e85aabf7bf 100644 --- a/vllm/v1/kv_offload/tiering/p2p/data/nixl.py +++ b/vllm/v1/kv_offload/tiering/p2p/data/nixl.py @@ -228,10 +228,7 @@ class NixlTransport(DataTransport): nixl_name = self._peer_nixl_names.pop(peer_id, None) dlist = self._remote_dlists.pop(peer_id, None) if self._agent is not None: - if dlist is not None: - self._agent.release_dlist_handle(dlist) - if nixl_name: - self._agent.remove_remote_agent(nixl_name) + self._release_remote_handles(peer_id, dlist, nixl_name) def _begin_registration(self, peer_id: str) -> int: """Claim the next registration generation for *peer_id*.""" @@ -287,18 +284,40 @@ class NixlTransport(DataTransport): peer_id, ) if self._agent is not None: + self._release_remote_handles(peer_id, remote_dlist, nixl_name) + return False + + def _release_remote_handles( + self, peer_id: str, dlist: object | None, nixl_name: str | None + ) -> None: + """Release a peer's descriptor list and remote agent independently. + + The two teardown operations are independent: NIXL may reject one + after a transport failure, and a ``release_dlist_handle`` that + raises must not skip ``remove_remote_agent``. Skipping it would + leave a stale remote registration behind after the peer is gone + (peer removal) or after its registration was superseded. + """ + if dlist is not None: try: - self._agent.release_dlist_handle(remote_dlist) - self._agent.remove_remote_agent(nixl_name) + self._agent.release_dlist_handle(dlist) except Exception as exc: logger.warning( - "NixlTransport %s: releasing superseded registration " - "for peer=%s failed: %s", + "NixlTransport %s: releasing dlist handle for peer=%s failed: %s", + self._agent_name, + peer_id, + exc, + ) + if nixl_name: + try: + self._agent.remove_remote_agent(nixl_name) + except Exception as exc: + logger.warning( + "NixlTransport %s: removing remote agent for peer=%s failed: %s", self._agent_name, peer_id, exc, ) - return False # ------------------------------------------------------------------ # Transfer submission and polling