[Bugfix] Fix CPU memory leak related to not cleaning up old remotes data (#44424)
Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -20,6 +20,7 @@ import torch
|
||||
from vllm import LLM
|
||||
from vllm.config import KVTransferConfig, set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
EngineTransferInfo,
|
||||
KVOutputAggregator,
|
||||
TransferTopology,
|
||||
get_current_attn_backend,
|
||||
@@ -1844,6 +1845,11 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
|
||||
worker.src_xfer_handles_by_tp_ratio = {-2: [456, 457]}
|
||||
worker.dst_xfer_side_handles = {"engine1": {0: 789}}
|
||||
worker._remote_agents = {"engine1": {0: "agent1"}}
|
||||
# _cleanup_remote_engine (called by shutdown) also clears these:
|
||||
worker.kv_caches_base_addr["engine1"] = {0: [0xABC]}
|
||||
worker.dst_num_blocks["engine1"] = 50
|
||||
worker.tp_mappings["engine1"] = MagicMock()
|
||||
worker._engine_last_active["engine1"] = time.perf_counter()
|
||||
worker._registered_descs = ["desc1", "desc2"]
|
||||
|
||||
mock_listener.is_alive.return_value = False
|
||||
@@ -1874,6 +1880,118 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
|
||||
mock_dereg.assert_any_call("desc2")
|
||||
|
||||
|
||||
# ── TTL-based remote engine eviction tests ──────────────────────────
|
||||
|
||||
|
||||
def _setup_worker_with_remote_engine(
|
||||
engine_ttl: float = 10.0,
|
||||
) -> tuple[Any, str]:
|
||||
"""Create a worker with one remote engine registered."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config={"engine_ttl": engine_ttl},
|
||||
)
|
||||
worker = NixlConnectorWorker(
|
||||
vllm_config,
|
||||
vllm_config.kv_transfer_config.engine_id,
|
||||
make_kv_cache_config(block_size=16),
|
||||
)
|
||||
|
||||
engine_id = "remote-engine-1"
|
||||
worker._remote_agents[engine_id] = {0: "agent_0", 1: "agent_1"}
|
||||
worker.dst_xfer_side_handles[engine_id] = {0: 100, 1: 200}
|
||||
worker.kv_caches_base_addr[engine_id] = {0: [0xABC]}
|
||||
worker.dst_num_blocks[engine_id] = 50
|
||||
worker.tp_mappings[engine_id] = MagicMock()
|
||||
worker._engine_last_active[engine_id] = time.perf_counter()
|
||||
|
||||
worker.transfer_topo = MagicMock()
|
||||
|
||||
return worker, engine_id
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
)
|
||||
def test_engine_ttl_eviction(default_vllm_config, dist_init):
|
||||
"""Stale engines are evicted when TTL expires."""
|
||||
worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=10.0)
|
||||
nixl_wrapper = worker.nixl_wrapper
|
||||
|
||||
with (
|
||||
patch.object(nixl_wrapper, "release_dlist_handle") as mock_rel,
|
||||
patch.object(nixl_wrapper, "remove_remote_agent") as mock_rem,
|
||||
):
|
||||
# Make the engine stale.
|
||||
worker._engine_last_active[engine_id] = time.perf_counter() - 20.0
|
||||
|
||||
worker._evict_stale_engines()
|
||||
|
||||
assert engine_id not in worker._remote_agents
|
||||
assert engine_id not in worker.dst_xfer_side_handles
|
||||
assert engine_id not in worker.kv_caches_base_addr
|
||||
assert engine_id not in worker.dst_num_blocks
|
||||
assert engine_id not in worker.tp_mappings
|
||||
assert engine_id not in worker._engine_last_active
|
||||
worker.transfer_topo.unregister_remote_engine.assert_called_with(engine_id)
|
||||
|
||||
assert mock_rel.call_count == 2
|
||||
mock_rel.assert_any_call(100)
|
||||
mock_rel.assert_any_call(200)
|
||||
|
||||
assert mock_rem.call_count == 2
|
||||
mock_rem.assert_any_call("agent_0")
|
||||
mock_rem.assert_any_call("agent_1")
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
)
|
||||
def test_engine_ttl_disabled(default_vllm_config, dist_init):
|
||||
"""Eviction is disabled when engine_ttl <= 0."""
|
||||
worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=0.0)
|
||||
|
||||
# Make the engine stale.
|
||||
worker._engine_last_active[engine_id] = time.perf_counter() - 9999.0
|
||||
|
||||
worker._evict_stale_engines()
|
||||
|
||||
# Nothing should be evicted.
|
||||
assert engine_id in worker._remote_agents
|
||||
assert engine_id in worker.dst_xfer_side_handles
|
||||
|
||||
|
||||
def test_transfer_topology_unregister():
|
||||
"""TransferTopology.unregister_remote_engine removes the engine."""
|
||||
topo = TransferTopology(
|
||||
tp_rank=0,
|
||||
tp_size=1,
|
||||
block_size=16,
|
||||
engine_id="local",
|
||||
is_mla=False,
|
||||
is_mamba=False,
|
||||
total_num_kv_heads=4,
|
||||
attn_backends=[FlashAttentionBackend],
|
||||
)
|
||||
|
||||
info = EngineTransferInfo(
|
||||
remote_tp_size=1,
|
||||
remote_block_size=16,
|
||||
remote_block_len=64,
|
||||
remote_physical_blocks_per_logical=1,
|
||||
)
|
||||
topo.register_remote_engine("remote-1", info)
|
||||
assert topo.get_engine_info("remote-1") is info
|
||||
|
||||
topo.unregister_remote_engine("remote-1")
|
||||
with pytest.raises(KeyError):
|
||||
topo.get_engine_info("remote-1")
|
||||
|
||||
# Idempotent: no error on double-unregister
|
||||
topo.unregister_remote_engine("remote-1")
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
|
||||
@@ -162,6 +162,7 @@ def test_read_blocks_for_req_expands_remote_ids(
|
||||
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker._physical_blocks_per_logical_kv_block = local_physical_per_logical
|
||||
worker._engine_last_active = {}
|
||||
|
||||
has_mamba = any(t is MambaSpec for t in resolved_types)
|
||||
has_swa = any(t is SlidingWindowSpec for t in resolved_types)
|
||||
|
||||
@@ -481,6 +481,11 @@ class TransferTopology:
|
||||
) -> EngineTransferInfo:
|
||||
return self._engines[(remote_engine_id, remote_pp_rank)]
|
||||
|
||||
def unregister_remote_engine(self, remote_engine_id: EngineId) -> None:
|
||||
# Remove all pp_rank entries for the remote engine.
|
||||
for key in [k for k in self._engines if k[0] == remote_engine_id]:
|
||||
del self._engines[key]
|
||||
|
||||
# ============================================================
|
||||
# Layout properties
|
||||
# ============================================================
|
||||
|
||||
@@ -408,6 +408,12 @@ class NixlConnectorWorker:
|
||||
# Protects _handshake_futures and _remote_agents.
|
||||
self._handshake_lock = threading.RLock()
|
||||
|
||||
# TTL-based eviction of stale remote engine state.
|
||||
self._engine_last_active: dict[EngineId, float] = {}
|
||||
self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"engine_ttl", 3600.0
|
||||
)
|
||||
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self.model_config = vllm_config.model_config
|
||||
|
||||
@@ -711,6 +717,7 @@ class NixlConnectorWorker:
|
||||
returned future.
|
||||
Failures to handshake are logged and the request is marked as failed.
|
||||
"""
|
||||
self._evict_stale_engines()
|
||||
with self._handshake_lock:
|
||||
if engine_id in self._remote_agents:
|
||||
return None
|
||||
@@ -731,6 +738,7 @@ class NixlConnectorWorker:
|
||||
del self._handshake_futures[eid]
|
||||
try:
|
||||
self._remote_agents[eid] = f.result()
|
||||
self._engine_last_active[eid] = time.perf_counter()
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="handshake_setup_failed",
|
||||
@@ -2028,6 +2036,9 @@ class NixlConnectorWorker:
|
||||
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
|
||||
assert meta.remote is not None and self.transfer_topo is not None
|
||||
engine_id = meta.remote.engine_id
|
||||
# Update last activity from this remote. Mind that cleanup is done on main
|
||||
# thread (this one), so we don't race on this structure.
|
||||
self._engine_last_active[engine_id] = time.perf_counter()
|
||||
plan = self.tp_mappings[engine_id]
|
||||
remote_info = self.transfer_topo.get_engine_info(engine_id)
|
||||
tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size)
|
||||
@@ -2473,6 +2484,61 @@ class NixlConnectorWorker:
|
||||
break
|
||||
return result
|
||||
|
||||
def _evict_stale_engines(self) -> None:
|
||||
"""Scan for and evict remote engines that have exceeded their TTL.
|
||||
|
||||
Called from the main thread in when a new remote engine appears.
|
||||
We can only go OOM as we discover and register a new remote, therefore we make
|
||||
sure we clean up stale engine data structures before then. This invariant
|
||||
prevents us from using background threads, though memory usage is not guaranteed
|
||||
to be "optimal" until a new handshake is performed.
|
||||
|
||||
Engines with active transfers or pending handshakes cannot be stale:
|
||||
- Active transfers touch _engine_last_active in start_load_kv.
|
||||
- Pending handshakes don't have an _engine_last_active entry yet
|
||||
"""
|
||||
# NOTE (NickLucche): This does NOT currently prevent OOMing if a huge number
|
||||
# of remote engines is registered all at once (adding a background cleanup
|
||||
# thread wouldnt help either).
|
||||
# If that scenario is plausible, we can follow up with an LRU eviction policy.
|
||||
if self._engine_ttl <= 0:
|
||||
return
|
||||
|
||||
now = time.perf_counter()
|
||||
for eid, last_active in list(self._engine_last_active.items()):
|
||||
if now - last_active > self._engine_ttl:
|
||||
self._cleanup_remote_engine(eid)
|
||||
|
||||
def _cleanup_remote_engine(
|
||||
self, engine_id: EngineId, *, log_eviction: bool = True
|
||||
) -> None:
|
||||
"""Remove all state for a single remote engine.
|
||||
|
||||
Releases NIXL resources (dlist handles, remote agents) and clears
|
||||
all per-engine data structures. Used by both TTL eviction and
|
||||
shutdown.
|
||||
"""
|
||||
assert engine_id in self._remote_agents
|
||||
|
||||
for handle in self.dst_xfer_side_handles.pop(engine_id).values():
|
||||
self.nixl_wrapper.release_dlist_handle(handle)
|
||||
for agent_name in self._remote_agents.pop(engine_id).values():
|
||||
self.nixl_wrapper.remove_remote_agent(agent_name)
|
||||
|
||||
del self.kv_caches_base_addr[engine_id]
|
||||
del self.dst_num_blocks[engine_id]
|
||||
del self.tp_mappings[engine_id]
|
||||
if self.transfer_topo is not None:
|
||||
self.transfer_topo.unregister_remote_engine(engine_id)
|
||||
|
||||
last_active = self._engine_last_active.pop(engine_id)
|
||||
if log_eviction:
|
||||
logger.info(
|
||||
"Evicted stale remote engine %s (inactive for %.1fs).",
|
||||
engine_id,
|
||||
time.perf_counter() - last_active,
|
||||
)
|
||||
|
||||
def __del__(self):
|
||||
self.shutdown()
|
||||
|
||||
@@ -2493,14 +2559,8 @@ class NixlConnectorWorker:
|
||||
for handle in handles:
|
||||
self.nixl_wrapper.release_dlist_handle(handle)
|
||||
self.src_xfer_handles_by_tp_ratio.clear()
|
||||
for dst_xfer_side_handles in self.dst_xfer_side_handles.values():
|
||||
for dst_xfer_side_handle in dst_xfer_side_handles.values():
|
||||
self.nixl_wrapper.release_dlist_handle(dst_xfer_side_handle)
|
||||
self.dst_xfer_side_handles.clear()
|
||||
for remote_agents in self._remote_agents.values():
|
||||
for agent_name in remote_agents.values():
|
||||
self.nixl_wrapper.remove_remote_agent(agent_name)
|
||||
self._remote_agents.clear()
|
||||
for engine_id in list(self._remote_agents):
|
||||
self._cleanup_remote_engine(engine_id, log_eviction=False)
|
||||
for desc in self._registered_descs:
|
||||
self.nixl_wrapper.deregister_memory(desc)
|
||||
self._registered_descs.clear()
|
||||
|
||||
Reference in New Issue
Block a user