diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 760a21c4c8..282458d1a9 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -2444,6 +2444,48 @@ def test_engine_ttl_eviction(default_vllm_config, dist_init): mock_rem.assert_any_call("agent_1") +@patch( + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", + FakeNixlWrapper, +) +def test_engine_with_inflight_transfer_is_not_evicted(default_vllm_config, dist_init): + """A transfer that outlives the TTL must keep its engine registered. + + _engine_last_active is stamped when a read is issued and not refreshed + while it runs, so a stalled transfer -- a peer that has lost its NIC holds + one indefinitely -- leaves its engine looking idle. Evicting it releases + the dlist handle and remote agent the transfer is still reading through. + """ + worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=10.0) + nixl_wrapper = worker.nixl_wrapper + + request_id = "req-still-reading" + worker._recving_transfers[request_id] = [MagicMock()] + worker._recving_metadata[request_id] = MagicMock( + remote=MagicMock(engine_id=engine_id) + ) + + with ( + patch.object(nixl_wrapper, "release_dlist_handle") as mock_rel, + patch.object(nixl_wrapper, "remove_remote_agent") as mock_rem, + ): + worker._engine_last_active[engine_id] = time.perf_counter() - 20.0 + + worker._evict_stale_engines() + + assert engine_id in worker._remote_agents + assert engine_id in worker.dst_xfer_side_handles + mock_rel.assert_not_called() + mock_rem.assert_not_called() + + # Once the transfer is done the engine is stale like any other. + del worker._recving_transfers[request_id] + worker._evict_stale_engines() + + assert engine_id not in worker._remote_agents + assert mock_rem.call_count == 2 + + @patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index dd2e97c965..e4b46cb64a 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -3143,7 +3143,9 @@ class NixlBaseConnectorWorker: to be "optimal" until a new handshake is performed. Engines with active transfers or pending handshakes cannot be stale: - - Active transfers touch _engine_last_active in start_load_kv. + - Reads stamp _engine_last_active when they are issued, and engines a + transfer is still reading from are held back explicitly, since the + stamp is not refreshed while the read runs. - Pending handshakes don't have an _engine_last_active entry yet """ # NOTE (NickLucche): This does NOT currently prevent OOMing if a huge number @@ -3154,10 +3156,25 @@ class NixlBaseConnectorWorker: return now = time.perf_counter() + busy = self._engines_with_inflight_transfers() for eid, last_active in list(self._engine_last_active.items()): - if now - last_active > self._engine_ttl: + if now - last_active > self._engine_ttl and eid not in busy: self._cleanup_remote_engine(eid) + def _engines_with_inflight_transfers(self) -> set[EngineId]: + """Remote engines a transfer is still reading from. + + The timestamp is stamped when a read is issued and not refreshed while + it runs, so a transfer that outlives the TTL leaves its engine looking + idle. A peer that has lost its NIC holds one indefinitely. + """ + return { + meta.remote.engine_id + for req_id in self._recving_transfers + if (meta := self._recving_metadata.get(req_id)) is not None + and meta.remote is not None + } + def _cleanup_remote_engine( self, engine_id: EngineId, *, log_eviction: bool = True ) -> None: