[Bugfix] Fix CPU memory leak related to not cleaning up old remotes data (#44424)
Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -20,6 +20,7 @@ import torch
|
||||
from vllm import LLM
|
||||
from vllm.config import KVTransferConfig, set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
EngineTransferInfo,
|
||||
KVOutputAggregator,
|
||||
TransferTopology,
|
||||
get_current_attn_backend,
|
||||
@@ -1844,6 +1845,11 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
|
||||
worker.src_xfer_handles_by_tp_ratio = {-2: [456, 457]}
|
||||
worker.dst_xfer_side_handles = {"engine1": {0: 789}}
|
||||
worker._remote_agents = {"engine1": {0: "agent1"}}
|
||||
# _cleanup_remote_engine (called by shutdown) also clears these:
|
||||
worker.kv_caches_base_addr["engine1"] = {0: [0xABC]}
|
||||
worker.dst_num_blocks["engine1"] = 50
|
||||
worker.tp_mappings["engine1"] = MagicMock()
|
||||
worker._engine_last_active["engine1"] = time.perf_counter()
|
||||
worker._registered_descs = ["desc1", "desc2"]
|
||||
|
||||
mock_listener.is_alive.return_value = False
|
||||
@@ -1874,6 +1880,118 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
|
||||
mock_dereg.assert_any_call("desc2")
|
||||
|
||||
|
||||
# ── TTL-based remote engine eviction tests ──────────────────────────
|
||||
|
||||
|
||||
def _setup_worker_with_remote_engine(
|
||||
engine_ttl: float = 10.0,
|
||||
) -> tuple[Any, str]:
|
||||
"""Create a worker with one remote engine registered."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config={"engine_ttl": engine_ttl},
|
||||
)
|
||||
worker = NixlConnectorWorker(
|
||||
vllm_config,
|
||||
vllm_config.kv_transfer_config.engine_id,
|
||||
make_kv_cache_config(block_size=16),
|
||||
)
|
||||
|
||||
engine_id = "remote-engine-1"
|
||||
worker._remote_agents[engine_id] = {0: "agent_0", 1: "agent_1"}
|
||||
worker.dst_xfer_side_handles[engine_id] = {0: 100, 1: 200}
|
||||
worker.kv_caches_base_addr[engine_id] = {0: [0xABC]}
|
||||
worker.dst_num_blocks[engine_id] = 50
|
||||
worker.tp_mappings[engine_id] = MagicMock()
|
||||
worker._engine_last_active[engine_id] = time.perf_counter()
|
||||
|
||||
worker.transfer_topo = MagicMock()
|
||||
|
||||
return worker, engine_id
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
)
|
||||
def test_engine_ttl_eviction(default_vllm_config, dist_init):
|
||||
"""Stale engines are evicted when TTL expires."""
|
||||
worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=10.0)
|
||||
nixl_wrapper = worker.nixl_wrapper
|
||||
|
||||
with (
|
||||
patch.object(nixl_wrapper, "release_dlist_handle") as mock_rel,
|
||||
patch.object(nixl_wrapper, "remove_remote_agent") as mock_rem,
|
||||
):
|
||||
# Make the engine stale.
|
||||
worker._engine_last_active[engine_id] = time.perf_counter() - 20.0
|
||||
|
||||
worker._evict_stale_engines()
|
||||
|
||||
assert engine_id not in worker._remote_agents
|
||||
assert engine_id not in worker.dst_xfer_side_handles
|
||||
assert engine_id not in worker.kv_caches_base_addr
|
||||
assert engine_id not in worker.dst_num_blocks
|
||||
assert engine_id not in worker.tp_mappings
|
||||
assert engine_id not in worker._engine_last_active
|
||||
worker.transfer_topo.unregister_remote_engine.assert_called_with(engine_id)
|
||||
|
||||
assert mock_rel.call_count == 2
|
||||
mock_rel.assert_any_call(100)
|
||||
mock_rel.assert_any_call(200)
|
||||
|
||||
assert mock_rem.call_count == 2
|
||||
mock_rem.assert_any_call("agent_0")
|
||||
mock_rem.assert_any_call("agent_1")
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
)
|
||||
def test_engine_ttl_disabled(default_vllm_config, dist_init):
|
||||
"""Eviction is disabled when engine_ttl <= 0."""
|
||||
worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=0.0)
|
||||
|
||||
# Make the engine stale.
|
||||
worker._engine_last_active[engine_id] = time.perf_counter() - 9999.0
|
||||
|
||||
worker._evict_stale_engines()
|
||||
|
||||
# Nothing should be evicted.
|
||||
assert engine_id in worker._remote_agents
|
||||
assert engine_id in worker.dst_xfer_side_handles
|
||||
|
||||
|
||||
def test_transfer_topology_unregister():
|
||||
"""TransferTopology.unregister_remote_engine removes the engine."""
|
||||
topo = TransferTopology(
|
||||
tp_rank=0,
|
||||
tp_size=1,
|
||||
block_size=16,
|
||||
engine_id="local",
|
||||
is_mla=False,
|
||||
is_mamba=False,
|
||||
total_num_kv_heads=4,
|
||||
attn_backends=[FlashAttentionBackend],
|
||||
)
|
||||
|
||||
info = EngineTransferInfo(
|
||||
remote_tp_size=1,
|
||||
remote_block_size=16,
|
||||
remote_block_len=64,
|
||||
remote_physical_blocks_per_logical=1,
|
||||
)
|
||||
topo.register_remote_engine("remote-1", info)
|
||||
assert topo.get_engine_info("remote-1") is info
|
||||
|
||||
topo.unregister_remote_engine("remote-1")
|
||||
with pytest.raises(KeyError):
|
||||
topo.get_engine_info("remote-1")
|
||||
|
||||
# Idempotent: no error on double-unregister
|
||||
topo.unregister_remote_engine("remote-1")
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
|
||||
@@ -162,6 +162,7 @@ def test_read_blocks_for_req_expands_remote_ids(
|
||||
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker._physical_blocks_per_logical_kv_block = local_physical_per_logical
|
||||
worker._engine_last_active = {}
|
||||
|
||||
has_mamba = any(t is MambaSpec for t in resolved_types)
|
||||
has_swa = any(t is SlidingWindowSpec for t in resolved_types)
|
||||
|
||||
Reference in New Issue
Block a user