[Bugfix][NIXL] Don't evict a remote engine a transfer is still reading from (#54689)

Signed-off-by: Yizheng Jiao <jyizheng@gmail.com>
This commit is contained in:
Yizheng Jiao
2026-09-11 18:06:09 -07:00
committed by GitHub
parent 46d2b23ac5
commit 56d7270138
2 changed files with 61 additions and 2 deletions
@@ -2444,6 +2444,48 @@ def test_engine_ttl_eviction(default_vllm_config, dist_init):
mock_rem.assert_any_call("agent_1")
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
)
def test_engine_with_inflight_transfer_is_not_evicted(default_vllm_config, dist_init):
"""A transfer that outlives the TTL must keep its engine registered.
_engine_last_active is stamped when a read is issued and not refreshed
while it runs, so a stalled transfer -- a peer that has lost its NIC holds
one indefinitely -- leaves its engine looking idle. Evicting it releases
the dlist handle and remote agent the transfer is still reading through.
"""
worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=10.0)
nixl_wrapper = worker.nixl_wrapper
request_id = "req-still-reading"
worker._recving_transfers[request_id] = [MagicMock()]
worker._recving_metadata[request_id] = MagicMock(
remote=MagicMock(engine_id=engine_id)
)
with (
patch.object(nixl_wrapper, "release_dlist_handle") as mock_rel,
patch.object(nixl_wrapper, "remove_remote_agent") as mock_rem,
):
worker._engine_last_active[engine_id] = time.perf_counter() - 20.0
worker._evict_stale_engines()
assert engine_id in worker._remote_agents
assert engine_id in worker.dst_xfer_side_handles
mock_rel.assert_not_called()
mock_rem.assert_not_called()
# Once the transfer is done the engine is stale like any other.
del worker._recving_transfers[request_id]
worker._evict_stale_engines()
assert engine_id not in worker._remote_agents
assert mock_rem.call_count == 2
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
@@ -3143,7 +3143,9 @@ class NixlBaseConnectorWorker:
to be "optimal" until a new handshake is performed.
Engines with active transfers or pending handshakes cannot be stale:
- Active transfers touch _engine_last_active in start_load_kv.
- Reads stamp _engine_last_active when they are issued, and engines a
transfer is still reading from are held back explicitly, since the
stamp is not refreshed while the read runs.
- Pending handshakes don't have an _engine_last_active entry yet
"""
# NOTE (NickLucche): This does NOT currently prevent OOMing if a huge number
@@ -3154,10 +3156,25 @@ class NixlBaseConnectorWorker:
return
now = time.perf_counter()
busy = self._engines_with_inflight_transfers()
for eid, last_active in list(self._engine_last_active.items()):
if now - last_active > self._engine_ttl:
if now - last_active > self._engine_ttl and eid not in busy:
self._cleanup_remote_engine(eid)
def _engines_with_inflight_transfers(self) -> set[EngineId]:
"""Remote engines a transfer is still reading from.
The timestamp is stamped when a read is issued and not refreshed while
it runs, so a transfer that outlives the TTL leaves its engine looking
idle. A peer that has lost its NIC holds one indefinitely.
"""
return {
meta.remote.engine_id
for req_id in self._recving_transfers
if (meta := self._recving_metadata.get(req_id)) is not None
and meta.remote is not None
}
def _cleanup_remote_engine(
self, engine_id: EngineId, *, log_eviction: bool = True
) -> None: