[Bugfix][NIXL] Release a dead peer's NIXL state without waiting for TTL (#50047)

Signed-off-by: Yannik Hinteregger <37209495+YannikHinteregger@users.noreply.github.com>
Co-authored-by: xijiade.aihemaiti <3146335281@qq.com>
This commit is contained in:
YannikHinteregger
2026-09-26 20:30:09 -07:00
committed by GitHub
co-authored by xijiade.aihemaiti
parent 3137ff0773
commit fff03267da
5 changed files with 237 additions and 3 deletions
@@ -7,6 +7,7 @@ import os
import queue import queue
import tempfile import tempfile
import textwrap import textwrap
import threading
import time import time
import uuid import uuid
from collections import defaultdict from collections import defaultdict
@@ -49,6 +50,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
NixlKVConnectorStats, NixlKVConnectorStats,
) )
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
RemoteMeta,
ReqMeta,
compute_nixl_compatibility_hash, compute_nixl_compatibility_hash,
) )
from vllm.distributed.kv_transfer.kv_transfer_state import ( from vllm.distributed.kv_transfer.kv_transfer_state import (
@@ -2087,6 +2090,10 @@ def recv_worker():
worker._recving_transfers = defaultdict(list) worker._recving_transfers = defaultdict(list)
worker._failed_recv_reqs = queue.Queue() worker._failed_recv_reqs = queue.Queue()
worker._recv_failures = set() worker._recv_failures = set()
worker._handshake_lock = threading.RLock()
worker._handshake_futures = {}
worker._remote_agents = {}
worker._engine_by_address = {}
worker._replicated_pcp_done_sending = set() worker._replicated_pcp_done_sending = set()
worker._invalid_block_ids = queue.Queue() worker._invalid_block_ids = queue.Queue()
worker._pending_recv_notifs = {} worker._pending_recv_notifs = {}
@@ -2681,6 +2688,174 @@ def test_engine_ttl_disabled(default_vllm_config, dist_init):
assert engine_id in worker.dst_xfer_side_handles assert engine_id in worker.dst_xfer_side_handles
@pytest.mark.cpu_test
class TestPeerReplacement:
@pytest.fixture(autouse=True)
def setup(self, monkeypatch):
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
config = create_vllm_config(kv_connector_extra_config={"engine_ttl": 0})
config.kv_transfer_config.kv_buffer_device = "cpu"
platform = SimpleNamespace(
device_type="cpu",
discover_numa_topology=lambda: [],
get_nixl_memory_type=lambda: "DRAM",
is_rocm=lambda: False,
)
with (
patch.object(bw, "NixlWrapper", FakeNixlWrapper),
patch.object(bw, "current_platform", platform),
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
patch.object(
bw, "get_current_attn_backends", return_value=[FlashAttentionBackend]
),
):
self.worker = FakeNixlConnectorWorker(config, "local", hand_shake_latency=0)
self.nixl = self.worker.nixl_wrapper
self.transport = MagicMock(wraps=self.nixl)
self.worker.nixl_wrapper = self.transport
monkeypatch.setattr(
self.worker._handshake_initiation_executor,
"submit",
lambda *args: Future(),
)
self._connect("old")
yield
self.worker.shutdown()
def _connect(self, engine_id="new", host="localhost", port=1234):
future = self.worker._ensure_handshake(engine_id, host, port, 1)
self.worker.REMOTE_ENGINE_ID = engine_id
self.nixl.REMOTE_AGENT_NAME = engine_id
future.set_result(self.worker._nixl_handshake(host, port, 1, engine_id))
def _request(self, engine_id="new", blocks=(1,), awaiting_kvs=True):
metadata = NixlConnectorMetadata()
metadata.reqs_to_recv[f"{engine_id}-req"] = ReqMeta(
local_block_ids=(blocks,),
local_physical_block_ids=(blocks,),
tp_size=1,
remote=RemoteMeta(
([2],), "localhost", 1234, engine_id, f"{engine_id}-prefill"
),
awaiting_kvs=awaiting_kvs,
)
return metadata
def test_cleans_old_peer_after_confirmed_handshake(self):
self._connect("healthy", host="other-host")
old_handle = self.worker.dst_xfer_side_handles["old"][0]
self.worker.start_load_kv(self._request())
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_not_called()
self._connect()
self.worker.get_transfer_results()
self.transport.release_dlist_handle.assert_called_once_with(old_handle)
self.transport.remove_remote_agent.assert_called_once_with("old")
assert set(self.worker._remote_agents) == {"new", "healthy"}
assert self.worker._engine_by_address == {
("localhost", 1234): "new",
("other-host", 1234): "healthy",
}
with pytest.raises(KeyError):
self.worker.transfer_topo.get_engine_info("old")
self.worker.start_load_kv(NixlConnectorMetadata())
assert self.worker.get_transfer_results().finished_recving == {"new-req"}
self.transport.remove_remote_agent.assert_called_once_with("old")
@pytest.mark.parametrize("host,port", [("other-host", 1234), ("localhost", 5678)])
def test_preserves_other_addresses(self, host, port):
self._connect(host=host, port=port)
self.worker.get_transfer_results()
assert "old" in self.worker._remote_agents
self.transport.remove_remote_agent.assert_not_called()
def test_preserves_old_peer_when_handshake_fails(self):
self.worker.start_load_kv(self._request())
self.worker._handshake_futures["new"].set_exception(
RuntimeError("handshake failed")
)
assert self.worker.get_transfer_results().failed_recving == {"new-req"}
assert "old" in self.worker._remote_agents
assert self.worker._engine_by_address == {("localhost", 1234): "old"}
self.transport.remove_remote_agent.assert_not_called()
@pytest.mark.parametrize(
"states,failed",
[
(["PROC", "DONE"], False),
(["ERR", "PROC", "DONE"], True),
(["ERR", "PROC", "ERR"], True),
],
ids=["healthy", "failed-sibling-completes", "failed-sibling-fails"],
)
def test_waits_for_reads_before_cleanup_and_failure_reporting(self, states, failed):
self.worker._recving_metadata.update(self._request("old").reqs_to_recv)
self.worker._recving_transfers["old-req"] = [1, 2] if failed else [1]
self.transport.check_xfer_state.side_effect = states
self._connect()
result = self.worker.get_transfer_results()
assert result.finished_recving == result.failed_recving == set()
assert self.worker.get_block_ids_with_load_errors() == set()
self.transport.remove_remote_agent.assert_not_called()
result = self.worker.get_transfer_results()
assert result.finished_recving == {"old-req"}
assert result.failed_recving == ({"old-req"} if failed else set())
assert self.worker.get_block_ids_with_load_errors() == (
{1} if failed else set()
)
assert self.transport.release_xfer_handle.call_count == (2 if failed else 1)
self.transport.remove_remote_agent.assert_called_once_with("old")
def test_waits_for_queued_notification(self):
metadata = self._request("old", blocks=())
self.worker._recving_metadata.update(metadata.reqs_to_recv)
self.worker._background_nixl_handshake(
"old-req", "old", metadata.reqs_to_recv["old-req"]
)
self._connect()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_not_called()
self.worker.start_load_kv(NixlConnectorMetadata())
self.transport.send_notif.assert_called_once_with(
"old", notif_msg=b"old-prefill:1"
)
result = self.worker.get_transfer_results()
assert result.finished_recving == {"old-req"}
assert result.failed_recving == set()
self.transport.remove_remote_agent.assert_called_once_with("old")
def test_waits_for_other_handshakes(self):
self.worker._ensure_handshake("other", "other-host", 1234, 1)
self._connect()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_not_called()
self._connect("other", host="other-host")
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
def test_rehandshakes_queued_request_for_released_engine(self):
# A request queued after its engine's handshake, but drained only after
# that engine was replaced and released, must not read from it.
metadata = self._request("old", blocks=(), awaiting_kvs=False)
self.worker._ready_requests.put(("old-req", metadata.reqs_to_recv["old-req"]))
self._connect()
self.worker.get_transfer_results()
self.transport.remove_remote_agent.assert_called_once_with("old")
self.worker.start_load_kv(NixlConnectorMetadata())
assert "old" in self.worker._handshake_futures
self.transport.send_notif.assert_not_called()
def test_transfer_topology_unregister(): def test_transfer_topology_unregister():
"""TransferTopology.unregister_remote_engine removes the engine.""" """TransferTopology.unregister_remote_engine removes the engine."""
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
@@ -4,6 +4,7 @@
import gc import gc
import queue import queue
import threading
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
@@ -54,6 +55,9 @@ def region_pull_worker():
worker.enable_permute_local_kv = False worker.enable_permute_local_kv = False
worker.enable_heterogeneous_attn_post_process = False worker.enable_heterogeneous_attn_post_process = False
worker._engine_last_active = {} worker._engine_last_active = {}
worker._handshake_lock = threading.RLock()
worker._handshake_futures = {}
worker._transfer_layer_group_ids = ()
worker._bidirectional_kv_xfer_enabled = False worker._bidirectional_kv_xfer_enabled = False
worker._recving_transfers = {} worker._recving_transfers = {}
worker.use_mla, worker._has_mamba = True, False worker.use_mla, worker._has_mamba = True, False
@@ -94,6 +98,7 @@ def region_pull_worker():
worker.dst_num_blocks = {"P": 100, "D": 100} worker.dst_num_blocks = {"P": 100, "D": 100}
worker.dst_region_num_blocks = {"P": [100, 100], "D": [100, 100]} worker.dst_region_num_blocks = {"P": [100, 100], "D": [100, 100]}
worker._remote_agents = {"P": {(0, 0): "P-rank0"}} worker._remote_agents = {"P": {(0, 0): "P-rank0"}}
worker._engine_by_address = {("localhost", 1): "P"}
worker._read_blocks_mixed = MagicMock() worker._read_blocks_mixed = MagicMock()
worker.nixl_wrapper = MagicMock() worker.nixl_wrapper = MagicMock()
return worker return worker
@@ -395,6 +395,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w._group_spec_types = (FullAttentionSpec,) w._group_spec_types = (FullAttentionSpec,)
w._engine_ttl = 0.0 w._engine_ttl = 0.0
w._engine_last_active = {} w._engine_last_active = {}
w._engine_by_address = {}
# Track _do_start_push_kv invocations. # Track _do_start_push_kv invocations.
calls: list[tuple[str, Any, dict[str, Any]]] = [] calls: list[tuple[str, Any, dict[str, Any]]] = []
@@ -839,6 +839,8 @@ class NixlBaseConnectorWorker:
self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config( self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config(
"engine_ttl", 3600.0 "engine_ttl", 3600.0
) )
# Map of pull peer (host, port) -> engine_id currently serving it.
self._engine_by_address: dict[tuple[str, int], EngineId] = {}
self.model_config = vllm_config.model_config self.model_config = vllm_config.model_config
@@ -1300,6 +1302,7 @@ class NixlBaseConnectorWorker:
self._remote_agents[eid] = remote_agents self._remote_agents[eid] = remote_agents
self._engine_clock_offset[eid] = clock_offset self._engine_clock_offset[eid] = clock_offset
self._engine_last_active[eid] = time.perf_counter() self._engine_last_active[eid] = time.perf_counter()
self._track_remote_engine_replacement(eid, host, port)
except Exception as e: except Exception as e:
self._log_failure( self._log_failure(
failure_type="handshake_setup_failed", failure_type="handshake_setup_failed",
@@ -2953,6 +2956,7 @@ class NixlBaseConnectorWorker:
# Handle timeout to avoid stranding blocks on remote. # Handle timeout to avoid stranding blocks on remote.
self._reap_expired_send_leases(done_sending) self._reap_expired_send_leases(done_sending)
self._cleanup_replaced_remote_engines()
return KVConnectorTransferResults( return KVConnectorTransferResults(
finished_sending=done_sending, finished_sending=done_sending,
finished_recving=done_recving, finished_recving=done_recving,
@@ -3541,14 +3545,53 @@ class NixlBaseConnectorWorker:
and meta.remote is not None and meta.remote is not None
} }
def _track_remote_engine_replacement(
self, engine_id: EngineId, host: str, port: int
) -> None:
"""Record the engine currently serving a pull peer address."""
# TODO: Also handle push mode, which handshakes in both directions.
if self._TRANSFER_MODE != "pull":
return
previous = self._engine_by_address.get((host, port))
self._engine_by_address[(host, port)] = engine_id
if previous is None or previous == engine_id:
return
logger.info(
"Remote engine at %s:%d changed from %s to %s.",
host,
port,
previous,
engine_id,
)
def _cleanup_replaced_remote_engines(self) -> None:
"""Release replaced pull peers once requests and handshakes have drained."""
if self._TRANSFER_MODE != "pull":
return
with self._handshake_lock:
if self._handshake_futures:
return
replaced = self._remote_agents.keys() - self._engine_by_address.values()
if not replaced:
return
# _recving_metadata is only accessed from this (main) thread.
busy = {
meta.remote.engine_id
for meta in self._recving_metadata.values()
if meta.remote is not None
}
for engine_id in replaced - busy:
self._cleanup_remote_engine(engine_id, log_eviction=False)
logger.info("Released NIXL state for replaced remote engine %s.", engine_id)
def _cleanup_remote_engine( def _cleanup_remote_engine(
self, engine_id: EngineId, *, log_eviction: bool = True self, engine_id: EngineId, *, log_eviction: bool = True
) -> None: ) -> None:
"""Remove all state for a single remote engine. """Remove all state for a single remote engine.
Releases NIXL resources (dlist handles, remote agents) and clears Releases NIXL resources (dlist handles, remote agents) and clears
all per-engine data structures. Used by both TTL eviction and all per-engine data structures. Used by TTL eviction, replaced peer
shutdown. cleanup and shutdown.
""" """
assert engine_id in self._remote_agents assert engine_id in self._remote_agents
@@ -3558,6 +3601,9 @@ class NixlBaseConnectorWorker:
# Pop under the handshake lock; NIXL teardown stays outside it. # Pop under the handshake lock; NIXL teardown stays outside it.
with self._handshake_lock: with self._handshake_lock:
agents = self._remote_agents.pop(engine_id) agents = self._remote_agents.pop(engine_id)
for address, eid in list(self._engine_by_address.items()):
if eid == engine_id:
del self._engine_by_address[address]
for agent_name in agents.values(): for agent_name in agents.values():
self.nixl_wrapper.remove_remote_agent(agent_name) self.nixl_wrapper.remove_remote_agent(agent_name)
@@ -84,7 +84,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
# Start transfers for requests whose handshakes have now finished. # Start transfers for requests whose handshakes have now finished.
while not self._ready_requests.empty(): while not self._ready_requests.empty():
self._read_blocks_for_req(*self._ready_requests.get_nowait()) req_id, meta = self._ready_requests.get_nowait()
assert meta.remote is not None
if meta.remote.engine_id not in self._remote_agents:
# The engine was released after its handshake completed, so
# handshake again. This fails if the engine is gone.
self._background_nixl_handshake(req_id, meta.remote.engine_id, meta)
continue
self._read_blocks_for_req(req_id, meta)
if self.pcp_rank > 0 and not self.pcp_dcp_sharded: if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank # Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank