From 750aab5b8e7f81b8d6d8dac33237cfb45a1f1455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Thu, 11 Jun 2026 16:54:52 +0200 Subject: [PATCH] [Bugfix] Fix CPU memory leak related to not cleaning up old remotes data (#44424) Signed-off-by: NickLucche --- .../kv_connector/unit/test_nixl_connector.py | 118 ++++++++++++++++++ .../unit/test_nixl_connector_hma.py | 1 + .../kv_transfer/kv_connector/utils.py | 5 + .../kv_connector/v1/nixl/worker.py | 76 +++++++++-- 4 files changed, 192 insertions(+), 8 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 6f6d8b1ca9..a2a46684bb 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -20,6 +20,7 @@ import torch from vllm import LLM from vllm.config import KVTransferConfig, set_current_vllm_config from vllm.distributed.kv_transfer.kv_connector.utils import ( + EngineTransferInfo, KVOutputAggregator, TransferTopology, get_current_attn_backend, @@ -1844,6 +1845,11 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init): worker.src_xfer_handles_by_tp_ratio = {-2: [456, 457]} worker.dst_xfer_side_handles = {"engine1": {0: 789}} worker._remote_agents = {"engine1": {0: "agent1"}} + # _cleanup_remote_engine (called by shutdown) also clears these: + worker.kv_caches_base_addr["engine1"] = {0: [0xABC]} + worker.dst_num_blocks["engine1"] = 50 + worker.tp_mappings["engine1"] = MagicMock() + worker._engine_last_active["engine1"] = time.perf_counter() worker._registered_descs = ["desc1", "desc2"] mock_listener.is_alive.return_value = False @@ -1874,6 +1880,118 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init): mock_dereg.assert_any_call("desc2") +# ── TTL-based remote engine eviction tests ────────────────────────── + + +def _setup_worker_with_remote_engine( + engine_ttl: float = 10.0, +) -> tuple[Any, str]: + """Create a worker with one remote engine registered.""" + vllm_config = create_vllm_config( + kv_connector_extra_config={"engine_ttl": engine_ttl}, + ) + worker = NixlConnectorWorker( + vllm_config, + vllm_config.kv_transfer_config.engine_id, + make_kv_cache_config(block_size=16), + ) + + engine_id = "remote-engine-1" + worker._remote_agents[engine_id] = {0: "agent_0", 1: "agent_1"} + worker.dst_xfer_side_handles[engine_id] = {0: 100, 1: 200} + worker.kv_caches_base_addr[engine_id] = {0: [0xABC]} + worker.dst_num_blocks[engine_id] = 50 + worker.tp_mappings[engine_id] = MagicMock() + worker._engine_last_active[engine_id] = time.perf_counter() + + worker.transfer_topo = MagicMock() + + return worker, engine_id + + +@patch( + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + FakeNixlWrapper, +) +def test_engine_ttl_eviction(default_vllm_config, dist_init): + """Stale engines are evicted when TTL expires.""" + worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=10.0) + nixl_wrapper = worker.nixl_wrapper + + with ( + patch.object(nixl_wrapper, "release_dlist_handle") as mock_rel, + patch.object(nixl_wrapper, "remove_remote_agent") as mock_rem, + ): + # Make the engine stale. + worker._engine_last_active[engine_id] = time.perf_counter() - 20.0 + + worker._evict_stale_engines() + + assert engine_id not in worker._remote_agents + assert engine_id not in worker.dst_xfer_side_handles + assert engine_id not in worker.kv_caches_base_addr + assert engine_id not in worker.dst_num_blocks + assert engine_id not in worker.tp_mappings + assert engine_id not in worker._engine_last_active + worker.transfer_topo.unregister_remote_engine.assert_called_with(engine_id) + + assert mock_rel.call_count == 2 + mock_rel.assert_any_call(100) + mock_rel.assert_any_call(200) + + assert mock_rem.call_count == 2 + mock_rem.assert_any_call("agent_0") + mock_rem.assert_any_call("agent_1") + + +@patch( + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + FakeNixlWrapper, +) +def test_engine_ttl_disabled(default_vllm_config, dist_init): + """Eviction is disabled when engine_ttl <= 0.""" + worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=0.0) + + # Make the engine stale. + worker._engine_last_active[engine_id] = time.perf_counter() - 9999.0 + + worker._evict_stale_engines() + + # Nothing should be evicted. + assert engine_id in worker._remote_agents + assert engine_id in worker.dst_xfer_side_handles + + +def test_transfer_topology_unregister(): + """TransferTopology.unregister_remote_engine removes the engine.""" + topo = TransferTopology( + tp_rank=0, + tp_size=1, + block_size=16, + engine_id="local", + is_mla=False, + is_mamba=False, + total_num_kv_heads=4, + attn_backends=[FlashAttentionBackend], + ) + + info = EngineTransferInfo( + remote_tp_size=1, + remote_block_size=16, + remote_block_len=64, + remote_physical_blocks_per_logical=1, + ) + topo.register_remote_engine("remote-1", info) + assert topo.get_engine_info("remote-1") is info + + topo.unregister_remote_engine("remote-1") + with pytest.raises(KeyError): + topo.get_engine_info("remote-1") + + # Idempotent: no error on double-unregister + topo.unregister_remote_engine("remote-1") + + @patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", FakeNixlWrapper, diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 6e399db7b1..af043113ed 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -162,6 +162,7 @@ def test_read_blocks_for_req_expands_remote_ids( worker = object.__new__(NixlConnectorWorker) worker._physical_blocks_per_logical_kv_block = local_physical_per_logical + worker._engine_last_active = {} has_mamba = any(t is MambaSpec for t in resolved_types) has_swa = any(t is SlidingWindowSpec for t in resolved_types) diff --git a/vllm/distributed/kv_transfer/kv_connector/utils.py b/vllm/distributed/kv_transfer/kv_connector/utils.py index 0ab694b7e7..71c9db075c 100644 --- a/vllm/distributed/kv_transfer/kv_connector/utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/utils.py @@ -481,6 +481,11 @@ class TransferTopology: ) -> EngineTransferInfo: return self._engines[(remote_engine_id, remote_pp_rank)] + def unregister_remote_engine(self, remote_engine_id: EngineId) -> None: + # Remove all pp_rank entries for the remote engine. + for key in [k for k in self._engines if k[0] == remote_engine_id]: + del self._engines[key] + # ============================================================ # Layout properties # ============================================================ diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py index 806a87c582..e4b20c01f4 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py @@ -408,6 +408,12 @@ class NixlConnectorWorker: # Protects _handshake_futures and _remote_agents. self._handshake_lock = threading.RLock() + # TTL-based eviction of stale remote engine state. + self._engine_last_active: dict[EngineId, float] = {} + self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config( + "engine_ttl", 3600.0 + ) + self.block_size = vllm_config.cache_config.block_size self.model_config = vllm_config.model_config @@ -711,6 +717,7 @@ class NixlConnectorWorker: returned future. Failures to handshake are logged and the request is marked as failed. """ + self._evict_stale_engines() with self._handshake_lock: if engine_id in self._remote_agents: return None @@ -731,6 +738,7 @@ class NixlConnectorWorker: del self._handshake_futures[eid] try: self._remote_agents[eid] = f.result() + self._engine_last_active[eid] = time.perf_counter() except Exception as e: self._log_failure( failure_type="handshake_setup_failed", @@ -2028,6 +2036,9 @@ class NixlConnectorWorker: def _read_blocks_for_req(self, req_id: str, meta: ReqMeta): assert meta.remote is not None and self.transfer_topo is not None engine_id = meta.remote.engine_id + # Update last activity from this remote. Mind that cleanup is done on main + # thread (this one), so we don't race on this structure. + self._engine_last_active[engine_id] = time.perf_counter() plan = self.tp_mappings[engine_id] remote_info = self.transfer_topo.get_engine_info(engine_id) tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size) @@ -2473,6 +2484,61 @@ class NixlConnectorWorker: break return result + def _evict_stale_engines(self) -> None: + """Scan for and evict remote engines that have exceeded their TTL. + + Called from the main thread in when a new remote engine appears. + We can only go OOM as we discover and register a new remote, therefore we make + sure we clean up stale engine data structures before then. This invariant + prevents us from using background threads, though memory usage is not guaranteed + to be "optimal" until a new handshake is performed. + + Engines with active transfers or pending handshakes cannot be stale: + - Active transfers touch _engine_last_active in start_load_kv. + - Pending handshakes don't have an _engine_last_active entry yet + """ + # NOTE (NickLucche): This does NOT currently prevent OOMing if a huge number + # of remote engines is registered all at once (adding a background cleanup + # thread wouldnt help either). + # If that scenario is plausible, we can follow up with an LRU eviction policy. + if self._engine_ttl <= 0: + return + + now = time.perf_counter() + for eid, last_active in list(self._engine_last_active.items()): + if now - last_active > self._engine_ttl: + self._cleanup_remote_engine(eid) + + def _cleanup_remote_engine( + self, engine_id: EngineId, *, log_eviction: bool = True + ) -> None: + """Remove all state for a single remote engine. + + Releases NIXL resources (dlist handles, remote agents) and clears + all per-engine data structures. Used by both TTL eviction and + shutdown. + """ + assert engine_id in self._remote_agents + + for handle in self.dst_xfer_side_handles.pop(engine_id).values(): + self.nixl_wrapper.release_dlist_handle(handle) + for agent_name in self._remote_agents.pop(engine_id).values(): + self.nixl_wrapper.remove_remote_agent(agent_name) + + del self.kv_caches_base_addr[engine_id] + del self.dst_num_blocks[engine_id] + del self.tp_mappings[engine_id] + if self.transfer_topo is not None: + self.transfer_topo.unregister_remote_engine(engine_id) + + last_active = self._engine_last_active.pop(engine_id) + if log_eviction: + logger.info( + "Evicted stale remote engine %s (inactive for %.1fs).", + engine_id, + time.perf_counter() - last_active, + ) + def __del__(self): self.shutdown() @@ -2493,14 +2559,8 @@ class NixlConnectorWorker: for handle in handles: self.nixl_wrapper.release_dlist_handle(handle) self.src_xfer_handles_by_tp_ratio.clear() - for dst_xfer_side_handles in self.dst_xfer_side_handles.values(): - for dst_xfer_side_handle in dst_xfer_side_handles.values(): - self.nixl_wrapper.release_dlist_handle(dst_xfer_side_handle) - self.dst_xfer_side_handles.clear() - for remote_agents in self._remote_agents.values(): - for agent_name in remote_agents.values(): - self.nixl_wrapper.remove_remote_agent(agent_name) - self._remote_agents.clear() + for engine_id in list(self._remote_agents): + self._cleanup_remote_engine(engine_id, log_eviction=False) for desc in self._registered_descs: self.nixl_wrapper.deregister_memory(desc) self._registered_descs.clear()