[Bugfix] Fix CPU memory leak related to not cleaning up old remotes data (#44424)

Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
Nicolò Lucchesi
2026-06-11 07:54:52 -07:00
committed by GitHub
parent 5edf7ff489
commit 750aab5b8e
4 changed files with 192 additions and 8 deletions
@@ -20,6 +20,7 @@ import torch
from vllm import LLM
from vllm.config import KVTransferConfig, set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.utils import (
EngineTransferInfo,
KVOutputAggregator,
TransferTopology,
get_current_attn_backend,
@@ -1844,6 +1845,11 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
worker.src_xfer_handles_by_tp_ratio = {-2: [456, 457]}
worker.dst_xfer_side_handles = {"engine1": {0: 789}}
worker._remote_agents = {"engine1": {0: "agent1"}}
# _cleanup_remote_engine (called by shutdown) also clears these:
worker.kv_caches_base_addr["engine1"] = {0: [0xABC]}
worker.dst_num_blocks["engine1"] = 50
worker.tp_mappings["engine1"] = MagicMock()
worker._engine_last_active["engine1"] = time.perf_counter()
worker._registered_descs = ["desc1", "desc2"]
mock_listener.is_alive.return_value = False
@@ -1874,6 +1880,118 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
mock_dereg.assert_any_call("desc2")
# ── TTL-based remote engine eviction tests ──────────────────────────
def _setup_worker_with_remote_engine(
engine_ttl: float = 10.0,
) -> tuple[Any, str]:
"""Create a worker with one remote engine registered."""
vllm_config = create_vllm_config(
kv_connector_extra_config={"engine_ttl": engine_ttl},
)
worker = NixlConnectorWorker(
vllm_config,
vllm_config.kv_transfer_config.engine_id,
make_kv_cache_config(block_size=16),
)
engine_id = "remote-engine-1"
worker._remote_agents[engine_id] = {0: "agent_0", 1: "agent_1"}
worker.dst_xfer_side_handles[engine_id] = {0: 100, 1: 200}
worker.kv_caches_base_addr[engine_id] = {0: [0xABC]}
worker.dst_num_blocks[engine_id] = 50
worker.tp_mappings[engine_id] = MagicMock()
worker._engine_last_active[engine_id] = time.perf_counter()
worker.transfer_topo = MagicMock()
return worker, engine_id
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
FakeNixlWrapper,
)
def test_engine_ttl_eviction(default_vllm_config, dist_init):
"""Stale engines are evicted when TTL expires."""
worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=10.0)
nixl_wrapper = worker.nixl_wrapper
with (
patch.object(nixl_wrapper, "release_dlist_handle") as mock_rel,
patch.object(nixl_wrapper, "remove_remote_agent") as mock_rem,
):
# Make the engine stale.
worker._engine_last_active[engine_id] = time.perf_counter() - 20.0
worker._evict_stale_engines()
assert engine_id not in worker._remote_agents
assert engine_id not in worker.dst_xfer_side_handles
assert engine_id not in worker.kv_caches_base_addr
assert engine_id not in worker.dst_num_blocks
assert engine_id not in worker.tp_mappings
assert engine_id not in worker._engine_last_active
worker.transfer_topo.unregister_remote_engine.assert_called_with(engine_id)
assert mock_rel.call_count == 2
mock_rel.assert_any_call(100)
mock_rel.assert_any_call(200)
assert mock_rem.call_count == 2
mock_rem.assert_any_call("agent_0")
mock_rem.assert_any_call("agent_1")
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
FakeNixlWrapper,
)
def test_engine_ttl_disabled(default_vllm_config, dist_init):
"""Eviction is disabled when engine_ttl <= 0."""
worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=0.0)
# Make the engine stale.
worker._engine_last_active[engine_id] = time.perf_counter() - 9999.0
worker._evict_stale_engines()
# Nothing should be evicted.
assert engine_id in worker._remote_agents
assert engine_id in worker.dst_xfer_side_handles
def test_transfer_topology_unregister():
"""TransferTopology.unregister_remote_engine removes the engine."""
topo = TransferTopology(
tp_rank=0,
tp_size=1,
block_size=16,
engine_id="local",
is_mla=False,
is_mamba=False,
total_num_kv_heads=4,
attn_backends=[FlashAttentionBackend],
)
info = EngineTransferInfo(
remote_tp_size=1,
remote_block_size=16,
remote_block_len=64,
remote_physical_blocks_per_logical=1,
)
topo.register_remote_engine("remote-1", info)
assert topo.get_engine_info("remote-1") is info
topo.unregister_remote_engine("remote-1")
with pytest.raises(KeyError):
topo.get_engine_info("remote-1")
# Idempotent: no error on double-unregister
topo.unregister_remote_engine("remote-1")
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
FakeNixlWrapper,
@@ -162,6 +162,7 @@ def test_read_blocks_for_req_expands_remote_ids(
worker = object.__new__(NixlConnectorWorker)
worker._physical_blocks_per_logical_kv_block = local_physical_per_logical
worker._engine_last_active = {}
has_mamba = any(t is MambaSpec for t in resolved_types)
has_swa = any(t is SlidingWindowSpec for t in resolved_types)
@@ -481,6 +481,11 @@ class TransferTopology:
) -> EngineTransferInfo:
return self._engines[(remote_engine_id, remote_pp_rank)]
def unregister_remote_engine(self, remote_engine_id: EngineId) -> None:
# Remove all pp_rank entries for the remote engine.
for key in [k for k in self._engines if k[0] == remote_engine_id]:
del self._engines[key]
# ============================================================
# Layout properties
# ============================================================
@@ -408,6 +408,12 @@ class NixlConnectorWorker:
# Protects _handshake_futures and _remote_agents.
self._handshake_lock = threading.RLock()
# TTL-based eviction of stale remote engine state.
self._engine_last_active: dict[EngineId, float] = {}
self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config(
"engine_ttl", 3600.0
)
self.block_size = vllm_config.cache_config.block_size
self.model_config = vllm_config.model_config
@@ -711,6 +717,7 @@ class NixlConnectorWorker:
returned future.
Failures to handshake are logged and the request is marked as failed.
"""
self._evict_stale_engines()
with self._handshake_lock:
if engine_id in self._remote_agents:
return None
@@ -731,6 +738,7 @@ class NixlConnectorWorker:
del self._handshake_futures[eid]
try:
self._remote_agents[eid] = f.result()
self._engine_last_active[eid] = time.perf_counter()
except Exception as e:
self._log_failure(
failure_type="handshake_setup_failed",
@@ -2028,6 +2036,9 @@ class NixlConnectorWorker:
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
assert meta.remote is not None and self.transfer_topo is not None
engine_id = meta.remote.engine_id
# Update last activity from this remote. Mind that cleanup is done on main
# thread (this one), so we don't race on this structure.
self._engine_last_active[engine_id] = time.perf_counter()
plan = self.tp_mappings[engine_id]
remote_info = self.transfer_topo.get_engine_info(engine_id)
tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size)
@@ -2473,6 +2484,61 @@ class NixlConnectorWorker:
break
return result
def _evict_stale_engines(self) -> None:
"""Scan for and evict remote engines that have exceeded their TTL.
Called from the main thread in when a new remote engine appears.
We can only go OOM as we discover and register a new remote, therefore we make
sure we clean up stale engine data structures before then. This invariant
prevents us from using background threads, though memory usage is not guaranteed
to be "optimal" until a new handshake is performed.
Engines with active transfers or pending handshakes cannot be stale:
- Active transfers touch _engine_last_active in start_load_kv.
- Pending handshakes don't have an _engine_last_active entry yet
"""
# NOTE (NickLucche): This does NOT currently prevent OOMing if a huge number
# of remote engines is registered all at once (adding a background cleanup
# thread wouldnt help either).
# If that scenario is plausible, we can follow up with an LRU eviction policy.
if self._engine_ttl <= 0:
return
now = time.perf_counter()
for eid, last_active in list(self._engine_last_active.items()):
if now - last_active > self._engine_ttl:
self._cleanup_remote_engine(eid)
def _cleanup_remote_engine(
self, engine_id: EngineId, *, log_eviction: bool = True
) -> None:
"""Remove all state for a single remote engine.
Releases NIXL resources (dlist handles, remote agents) and clears
all per-engine data structures. Used by both TTL eviction and
shutdown.
"""
assert engine_id in self._remote_agents
for handle in self.dst_xfer_side_handles.pop(engine_id).values():
self.nixl_wrapper.release_dlist_handle(handle)
for agent_name in self._remote_agents.pop(engine_id).values():
self.nixl_wrapper.remove_remote_agent(agent_name)
del self.kv_caches_base_addr[engine_id]
del self.dst_num_blocks[engine_id]
del self.tp_mappings[engine_id]
if self.transfer_topo is not None:
self.transfer_topo.unregister_remote_engine(engine_id)
last_active = self._engine_last_active.pop(engine_id)
if log_eviction:
logger.info(
"Evicted stale remote engine %s (inactive for %.1fs).",
engine_id,
time.perf_counter() - last_active,
)
def __del__(self):
self.shutdown()
@@ -2493,14 +2559,8 @@ class NixlConnectorWorker:
for handle in handles:
self.nixl_wrapper.release_dlist_handle(handle)
self.src_xfer_handles_by_tp_ratio.clear()
for dst_xfer_side_handles in self.dst_xfer_side_handles.values():
for dst_xfer_side_handle in dst_xfer_side_handles.values():
self.nixl_wrapper.release_dlist_handle(dst_xfer_side_handle)
self.dst_xfer_side_handles.clear()
for remote_agents in self._remote_agents.values():
for agent_name in remote_agents.values():
self.nixl_wrapper.remove_remote_agent(agent_name)
self._remote_agents.clear()
for engine_id in list(self._remote_agents):
self._cleanup_remote_engine(engine_id, log_eviction=False)
for desc in self._registered_descs:
self.nixl_wrapper.deregister_memory(desc)
self._registered_descs.clear()