[Bugfix][KV Offload] Always attempt NIXL agent removal during peer teardown (#60279)

Signed-off-by: nej1gotnochill <divyanshlalotra2@gmail.com>
Co-authored-by: Codebuff <noreply@codebuff.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Divyansh Lalotra
2026-10-15 14:07:23 +03:00
committed by GitHub
co-authored by Codebuff Kevin H. Luu
parent 81e42c9987
commit 8b27d27402
2 changed files with 94 additions and 9 deletions
@@ -11,6 +11,7 @@ from unittest.mock import MagicMock, patch
import numpy as np
import pytest
from vllm.v1.kv_offload.tiering.p2p.data import nixl as nixl_module
from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult
from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport
@@ -425,6 +426,71 @@ class TestNixlTransportWithMockedAgent:
transport._agent.release_dlist_handle.assert_called()
transport._agent.remove_remote_agent.assert_called()
def test_remove_peer_removes_agent_even_if_release_fails(self):
"""A dlist release failure must not skip the agent removal.
Teardown runs two independent NIXL operations. If
release_dlist_handle() raises, remove_remote_agent() must still be
attempted with the registered agent name; skipping it would leave
a stale NIXL registration behind after the peer is gone.
"""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
transport._agent.release_dlist_handle.side_effect = RuntimeError(
"simulated release failure"
)
with patch.object(nixl_module.logger, "warning") as mock_warning:
transport.remove_remote_peer("peer:1")
# The release failure was logged rather than swallowed silently.
mock_warning.assert_called_once()
# Agent removal was still attempted with the registered name.
transport._agent.release_dlist_handle.assert_called_once()
transport._agent.remove_remote_agent.assert_called_once_with("nixl-peer-name")
assert "peer:1" not in transport._remote_dlists
def test_superseded_registration_removes_agent_even_if_release_fails(self):
"""A release failure while discarding must still remove the agent.
_publish_peer discards handles built under a stale generation when
the peer was removed or re-registered mid-registration. Its
descriptor-list release and agent removal are independent too: if
release_dlist_handle() raises, remove_remote_agent() must still be
attempted for the superseded registration, otherwise its remote
agent stays registered forever.
"""
transport = self._make_transport()
building = threading.Event()
release = threading.Event()
def blocking_prep(*args, **kwargs):
building.set()
release.wait(timeout=5)
return MagicMock(name="remote_dlist")
transport._agent.prep_xfer_dlist.side_effect = blocking_prep
future = transport.add_remote_peer_async("peer:1", b"meta", 0x1000, 8, 1024)
assert building.wait(timeout=5)
# Supersede the registration while its worker still builds handles.
transport.remove_remote_peer("peer:1")
transport._agent.release_dlist_handle.side_effect = RuntimeError(
"simulated release failure"
)
with patch.object(nixl_module.logger, "warning") as mock_warning:
release.set()
with pytest.raises(RuntimeError, match="superseded"):
future.result(timeout=5)
# The release failure was logged rather than swallowed silently.
mock_warning.assert_called_once()
# The superseded registration's agent was still removed.
transport._agent.release_dlist_handle.assert_called_once()
transport._agent.remove_remote_agent.assert_called_once_with("nixl-peer-name")
assert "peer:1" not in transport._remote_dlists
def test_close_releases_everything(self):
"""Close releases all handles and clears state."""
transport = self._make_transport()
+28 -9
View File
@@ -228,10 +228,7 @@ class NixlTransport(DataTransport):
nixl_name = self._peer_nixl_names.pop(peer_id, None)
dlist = self._remote_dlists.pop(peer_id, None)
if self._agent is not None:
if dlist is not None:
self._agent.release_dlist_handle(dlist)
if nixl_name:
self._agent.remove_remote_agent(nixl_name)
self._release_remote_handles(peer_id, dlist, nixl_name)
def _begin_registration(self, peer_id: str) -> int:
"""Claim the next registration generation for *peer_id*."""
@@ -287,18 +284,40 @@ class NixlTransport(DataTransport):
peer_id,
)
if self._agent is not None:
self._release_remote_handles(peer_id, remote_dlist, nixl_name)
return False
def _release_remote_handles(
self, peer_id: str, dlist: object | None, nixl_name: str | None
) -> None:
"""Release a peer's descriptor list and remote agent independently.
The two teardown operations are independent: NIXL may reject one
after a transport failure, and a ``release_dlist_handle`` that
raises must not skip ``remove_remote_agent``. Skipping it would
leave a stale remote registration behind after the peer is gone
(peer removal) or after its registration was superseded.
"""
if dlist is not None:
try:
self._agent.release_dlist_handle(remote_dlist)
self._agent.remove_remote_agent(nixl_name)
self._agent.release_dlist_handle(dlist)
except Exception as exc:
logger.warning(
"NixlTransport %s: releasing superseded registration "
"for peer=%s failed: %s",
"NixlTransport %s: releasing dlist handle for peer=%s failed: %s",
self._agent_name,
peer_id,
exc,
)
if nixl_name:
try:
self._agent.remove_remote_agent(nixl_name)
except Exception as exc:
logger.warning(
"NixlTransport %s: removing remote agent for peer=%s failed: %s",
self._agent_name,
peer_id,
exc,
)
return False
# ------------------------------------------------------------------
# Transfer submission and polling