[CI] Clean up remote servers on pytest parent exit (#41570)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
+80
-3
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import atexit
|
||||
import contextlib
|
||||
import copy
|
||||
import functools
|
||||
@@ -134,6 +135,11 @@ class RemoteVLLMServer:
|
||||
"""
|
||||
|
||||
DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
|
||||
_active_servers: set["RemoteVLLMServer"] = set()
|
||||
_active_servers_lock = threading.RLock()
|
||||
_cleanup_hooks_registered = False
|
||||
_signal_hooks_registered = False
|
||||
_previous_signal_handlers: dict[int, Any] = {}
|
||||
proc: subprocess.Popen
|
||||
|
||||
def _create_cli_subcommand(self):
|
||||
@@ -209,6 +215,7 @@ class RemoteVLLMServer:
|
||||
)
|
||||
|
||||
self._pre_download_model(model, args)
|
||||
self._shutdown_complete = False
|
||||
|
||||
# Record GPU memory before server start so we know what
|
||||
# "released" looks like.
|
||||
@@ -221,6 +228,7 @@ class RemoteVLLMServer:
|
||||
)
|
||||
|
||||
self._start_server(model, vllm_serve_args, env_dict)
|
||||
self._register_active_server()
|
||||
max_wait_seconds = max_wait_seconds or 480
|
||||
try:
|
||||
self._wait_for_server(url=self.url_for("health"), timeout=max_wait_seconds)
|
||||
@@ -246,8 +254,70 @@ class RemoteVLLMServer:
|
||||
(when the server fails to start). Must be safe to call even if
|
||||
the process is already dead.
|
||||
"""
|
||||
self._terminate_process_tree()
|
||||
self._wait_for_gpu_memory_release()
|
||||
if self._shutdown_complete:
|
||||
return
|
||||
|
||||
self._shutdown_complete = True
|
||||
try:
|
||||
self._terminate_process_tree()
|
||||
self._wait_for_gpu_memory_release()
|
||||
finally:
|
||||
self._unregister_active_server()
|
||||
|
||||
@classmethod
|
||||
def _ensure_cleanup_hooks_registered(cls) -> None:
|
||||
"""Register process-exit cleanup for detached server subprocesses."""
|
||||
root_cls = RemoteVLLMServer
|
||||
with root_cls._active_servers_lock:
|
||||
if not root_cls._cleanup_hooks_registered:
|
||||
atexit.register(root_cls._shutdown_active_servers)
|
||||
root_cls._cleanup_hooks_registered = True
|
||||
|
||||
if (
|
||||
threading.current_thread() is threading.main_thread()
|
||||
and not root_cls._signal_hooks_registered
|
||||
):
|
||||
for signum in (signal.SIGTERM, signal.SIGINT):
|
||||
root_cls._previous_signal_handlers[signum] = signal.getsignal(
|
||||
signum
|
||||
)
|
||||
signal.signal(signum, root_cls._handle_parent_signal)
|
||||
root_cls._signal_hooks_registered = True
|
||||
|
||||
def _register_active_server(self) -> None:
|
||||
"""Track this server so parent-process exits still clean it up."""
|
||||
RemoteVLLMServer._ensure_cleanup_hooks_registered()
|
||||
with RemoteVLLMServer._active_servers_lock:
|
||||
RemoteVLLMServer._active_servers.add(self)
|
||||
|
||||
def _unregister_active_server(self) -> None:
|
||||
with RemoteVLLMServer._active_servers_lock:
|
||||
RemoteVLLMServer._active_servers.discard(self)
|
||||
|
||||
@classmethod
|
||||
def _shutdown_active_servers(cls) -> None:
|
||||
"""Best-effort shutdown for all live RemoteVLLMServer instances."""
|
||||
with cls._active_servers_lock:
|
||||
servers = list(cls._active_servers)
|
||||
|
||||
for server in servers:
|
||||
with contextlib.suppress(Exception):
|
||||
server._shutdown()
|
||||
|
||||
@classmethod
|
||||
def _handle_parent_signal(cls, signum, frame) -> None:
|
||||
"""Clean up detached servers before letting the signal terminate pytest."""
|
||||
cls._shutdown_active_servers()
|
||||
|
||||
previous_handler = cls._previous_signal_handlers.get(signum, signal.SIG_DFL)
|
||||
if callable(previous_handler):
|
||||
previous_handler(signum, frame)
|
||||
elif previous_handler == signal.SIG_IGN:
|
||||
return
|
||||
elif signum == signal.SIGINT:
|
||||
raise KeyboardInterrupt
|
||||
else:
|
||||
raise SystemExit(128 + signum)
|
||||
|
||||
def _terminate_process_tree(self) -> None:
|
||||
"""Kill the server process tree without waiting for GPU memory release.
|
||||
@@ -315,6 +385,9 @@ class RemoteVLLMServer:
|
||||
if not servers:
|
||||
return
|
||||
|
||||
for server in servers:
|
||||
server._shutdown_complete = True
|
||||
|
||||
threads = [
|
||||
threading.Thread(
|
||||
target=s._terminate_process_tree,
|
||||
@@ -339,7 +412,11 @@ class RemoteVLLMServer:
|
||||
else s._pre_server_gpu_memory
|
||||
),
|
||||
)
|
||||
earliest._wait_for_gpu_memory_release()
|
||||
try:
|
||||
earliest._wait_for_gpu_memory_release()
|
||||
finally:
|
||||
for server in servers:
|
||||
server._unregister_active_server()
|
||||
|
||||
def _kill_process_group_survivors(
|
||||
self, pgid: int | None, timeout: float = 15.0
|
||||
|
||||
Reference in New Issue
Block a user