[Bugfix][KV Offload] Always attempt NIXL agent removal during peer teardown (#60279)
Signed-off-by: nej1gotnochill <divyanshlalotra2@gmail.com> Co-authored-by: Codebuff <noreply@codebuff.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Codebuff
Kevin H. Luu
parent
81e42c9987
commit
8b27d27402
@@ -11,6 +11,7 @@ from unittest.mock import MagicMock, patch
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.v1.kv_offload.tiering.p2p.data import nixl as nixl_module
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport
|
||||
|
||||
@@ -425,6 +426,71 @@ class TestNixlTransportWithMockedAgent:
|
||||
transport._agent.release_dlist_handle.assert_called()
|
||||
transport._agent.remove_remote_agent.assert_called()
|
||||
|
||||
def test_remove_peer_removes_agent_even_if_release_fails(self):
|
||||
"""A dlist release failure must not skip the agent removal.
|
||||
|
||||
Teardown runs two independent NIXL operations. If
|
||||
release_dlist_handle() raises, remove_remote_agent() must still be
|
||||
attempted with the registered agent name; skipping it would leave
|
||||
a stale NIXL registration behind after the peer is gone.
|
||||
"""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
transport._agent.release_dlist_handle.side_effect = RuntimeError(
|
||||
"simulated release failure"
|
||||
)
|
||||
|
||||
with patch.object(nixl_module.logger, "warning") as mock_warning:
|
||||
transport.remove_remote_peer("peer:1")
|
||||
|
||||
# The release failure was logged rather than swallowed silently.
|
||||
mock_warning.assert_called_once()
|
||||
# Agent removal was still attempted with the registered name.
|
||||
transport._agent.release_dlist_handle.assert_called_once()
|
||||
transport._agent.remove_remote_agent.assert_called_once_with("nixl-peer-name")
|
||||
assert "peer:1" not in transport._remote_dlists
|
||||
|
||||
def test_superseded_registration_removes_agent_even_if_release_fails(self):
|
||||
"""A release failure while discarding must still remove the agent.
|
||||
|
||||
_publish_peer discards handles built under a stale generation when
|
||||
the peer was removed or re-registered mid-registration. Its
|
||||
descriptor-list release and agent removal are independent too: if
|
||||
release_dlist_handle() raises, remove_remote_agent() must still be
|
||||
attempted for the superseded registration, otherwise its remote
|
||||
agent stays registered forever.
|
||||
"""
|
||||
transport = self._make_transport()
|
||||
building = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def blocking_prep(*args, **kwargs):
|
||||
building.set()
|
||||
release.wait(timeout=5)
|
||||
return MagicMock(name="remote_dlist")
|
||||
|
||||
transport._agent.prep_xfer_dlist.side_effect = blocking_prep
|
||||
|
||||
future = transport.add_remote_peer_async("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
assert building.wait(timeout=5)
|
||||
# Supersede the registration while its worker still builds handles.
|
||||
transport.remove_remote_peer("peer:1")
|
||||
transport._agent.release_dlist_handle.side_effect = RuntimeError(
|
||||
"simulated release failure"
|
||||
)
|
||||
|
||||
with patch.object(nixl_module.logger, "warning") as mock_warning:
|
||||
release.set()
|
||||
with pytest.raises(RuntimeError, match="superseded"):
|
||||
future.result(timeout=5)
|
||||
|
||||
# The release failure was logged rather than swallowed silently.
|
||||
mock_warning.assert_called_once()
|
||||
# The superseded registration's agent was still removed.
|
||||
transport._agent.release_dlist_handle.assert_called_once()
|
||||
transport._agent.remove_remote_agent.assert_called_once_with("nixl-peer-name")
|
||||
assert "peer:1" not in transport._remote_dlists
|
||||
|
||||
def test_close_releases_everything(self):
|
||||
"""Close releases all handles and clears state."""
|
||||
transport = self._make_transport()
|
||||
|
||||
@@ -228,10 +228,7 @@ class NixlTransport(DataTransport):
|
||||
nixl_name = self._peer_nixl_names.pop(peer_id, None)
|
||||
dlist = self._remote_dlists.pop(peer_id, None)
|
||||
if self._agent is not None:
|
||||
if dlist is not None:
|
||||
self._agent.release_dlist_handle(dlist)
|
||||
if nixl_name:
|
||||
self._agent.remove_remote_agent(nixl_name)
|
||||
self._release_remote_handles(peer_id, dlist, nixl_name)
|
||||
|
||||
def _begin_registration(self, peer_id: str) -> int:
|
||||
"""Claim the next registration generation for *peer_id*."""
|
||||
@@ -287,18 +284,40 @@ class NixlTransport(DataTransport):
|
||||
peer_id,
|
||||
)
|
||||
if self._agent is not None:
|
||||
self._release_remote_handles(peer_id, remote_dlist, nixl_name)
|
||||
return False
|
||||
|
||||
def _release_remote_handles(
|
||||
self, peer_id: str, dlist: object | None, nixl_name: str | None
|
||||
) -> None:
|
||||
"""Release a peer's descriptor list and remote agent independently.
|
||||
|
||||
The two teardown operations are independent: NIXL may reject one
|
||||
after a transport failure, and a ``release_dlist_handle`` that
|
||||
raises must not skip ``remove_remote_agent``. Skipping it would
|
||||
leave a stale remote registration behind after the peer is gone
|
||||
(peer removal) or after its registration was superseded.
|
||||
"""
|
||||
if dlist is not None:
|
||||
try:
|
||||
self._agent.release_dlist_handle(remote_dlist)
|
||||
self._agent.remove_remote_agent(nixl_name)
|
||||
self._agent.release_dlist_handle(dlist)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"NixlTransport %s: releasing superseded registration "
|
||||
"for peer=%s failed: %s",
|
||||
"NixlTransport %s: releasing dlist handle for peer=%s failed: %s",
|
||||
self._agent_name,
|
||||
peer_id,
|
||||
exc,
|
||||
)
|
||||
if nixl_name:
|
||||
try:
|
||||
self._agent.remove_remote_agent(nixl_name)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"NixlTransport %s: removing remote agent for peer=%s failed: %s",
|
||||
self._agent_name,
|
||||
peer_id,
|
||||
exc,
|
||||
)
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Transfer submission and polling
|
||||
|
||||
Reference in New Issue
Block a user