[Bugfix][NIXL] Release a dead peer's NIXL state without waiting for TTL (#50047)

Signed-off-by: Yannik Hinteregger <37209495+YannikHinteregger@users.noreply.github.com>
Co-authored-by: xijiade.aihemaiti <3146335281@qq.com>
This commit is contained in:
YannikHinteregger
2026-09-26 20:30:09 -07:00
committed by GitHub
co-authored by xijiade.aihemaiti
parent 3137ff0773
commit fff03267da
5 changed files with 237 additions and 3 deletions
@@ -839,6 +839,8 @@ class NixlBaseConnectorWorker:
self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config(
"engine_ttl", 3600.0
)
# Map of pull peer (host, port) -> engine_id currently serving it.
self._engine_by_address: dict[tuple[str, int], EngineId] = {}
self.model_config = vllm_config.model_config
@@ -1300,6 +1302,7 @@ class NixlBaseConnectorWorker:
self._remote_agents[eid] = remote_agents
self._engine_clock_offset[eid] = clock_offset
self._engine_last_active[eid] = time.perf_counter()
self._track_remote_engine_replacement(eid, host, port)
except Exception as e:
self._log_failure(
failure_type="handshake_setup_failed",
@@ -2953,6 +2956,7 @@ class NixlBaseConnectorWorker:
# Handle timeout to avoid stranding blocks on remote.
self._reap_expired_send_leases(done_sending)
self._cleanup_replaced_remote_engines()
return KVConnectorTransferResults(
finished_sending=done_sending,
finished_recving=done_recving,
@@ -3541,14 +3545,53 @@ class NixlBaseConnectorWorker:
and meta.remote is not None
}
def _track_remote_engine_replacement(
self, engine_id: EngineId, host: str, port: int
) -> None:
"""Record the engine currently serving a pull peer address."""
# TODO: Also handle push mode, which handshakes in both directions.
if self._TRANSFER_MODE != "pull":
return
previous = self._engine_by_address.get((host, port))
self._engine_by_address[(host, port)] = engine_id
if previous is None or previous == engine_id:
return
logger.info(
"Remote engine at %s:%d changed from %s to %s.",
host,
port,
previous,
engine_id,
)
def _cleanup_replaced_remote_engines(self) -> None:
"""Release replaced pull peers once requests and handshakes have drained."""
if self._TRANSFER_MODE != "pull":
return
with self._handshake_lock:
if self._handshake_futures:
return
replaced = self._remote_agents.keys() - self._engine_by_address.values()
if not replaced:
return
# _recving_metadata is only accessed from this (main) thread.
busy = {
meta.remote.engine_id
for meta in self._recving_metadata.values()
if meta.remote is not None
}
for engine_id in replaced - busy:
self._cleanup_remote_engine(engine_id, log_eviction=False)
logger.info("Released NIXL state for replaced remote engine %s.", engine_id)
def _cleanup_remote_engine(
self, engine_id: EngineId, *, log_eviction: bool = True
) -> None:
"""Remove all state for a single remote engine.
Releases NIXL resources (dlist handles, remote agents) and clears
all per-engine data structures. Used by both TTL eviction and
shutdown.
all per-engine data structures. Used by TTL eviction, replaced peer
cleanup and shutdown.
"""
assert engine_id in self._remote_agents
@@ -3558,6 +3601,9 @@ class NixlBaseConnectorWorker:
# Pop under the handshake lock; NIXL teardown stays outside it.
with self._handshake_lock:
agents = self._remote_agents.pop(engine_id)
for address, eid in list(self._engine_by_address.items()):
if eid == engine_id:
del self._engine_by_address[address]
for agent_name in agents.values():
self.nixl_wrapper.remove_remote_agent(agent_name)
@@ -84,7 +84,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
# Start transfers for requests whose handshakes have now finished.
while not self._ready_requests.empty():
self._read_blocks_for_req(*self._ready_requests.get_nowait())
req_id, meta = self._ready_requests.get_nowait()
assert meta.remote is not None
if meta.remote.engine_id not in self._remote_agents:
# The engine was released after its handshake completed, so
# handshake again. This fails if the engine is gone.
self._background_nixl_handshake(req_id, meta.remote.engine_id, meta)
continue
self._read_blocks_for_req(req_id, meta)
if self.pcp_rank > 0 and not self.pcp_dcp_sharded:
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank