[CI] Clean up remote servers on pytest parent exit (#41570)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-05-04 03:11:22 +00:00
committed by GitHub
parent 894a02500b
commit 67058ca326
+80 -3
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import atexit
import contextlib
import copy
import functools
@@ -134,6 +135,11 @@ class RemoteVLLMServer:
"""
DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key
_active_servers: set["RemoteVLLMServer"] = set()
_active_servers_lock = threading.RLock()
_cleanup_hooks_registered = False
_signal_hooks_registered = False
_previous_signal_handlers: dict[int, Any] = {}
proc: subprocess.Popen
def _create_cli_subcommand(self):
@@ -209,6 +215,7 @@ class RemoteVLLMServer:
)
self._pre_download_model(model, args)
self._shutdown_complete = False
# Record GPU memory before server start so we know what
# "released" looks like.
@@ -221,6 +228,7 @@ class RemoteVLLMServer:
)
self._start_server(model, vllm_serve_args, env_dict)
self._register_active_server()
max_wait_seconds = max_wait_seconds or 480
try:
self._wait_for_server(url=self.url_for("health"), timeout=max_wait_seconds)
@@ -246,8 +254,70 @@ class RemoteVLLMServer:
(when the server fails to start). Must be safe to call even if
the process is already dead.
"""
self._terminate_process_tree()
self._wait_for_gpu_memory_release()
if self._shutdown_complete:
return
self._shutdown_complete = True
try:
self._terminate_process_tree()
self._wait_for_gpu_memory_release()
finally:
self._unregister_active_server()
@classmethod
def _ensure_cleanup_hooks_registered(cls) -> None:
"""Register process-exit cleanup for detached server subprocesses."""
root_cls = RemoteVLLMServer
with root_cls._active_servers_lock:
if not root_cls._cleanup_hooks_registered:
atexit.register(root_cls._shutdown_active_servers)
root_cls._cleanup_hooks_registered = True
if (
threading.current_thread() is threading.main_thread()
and not root_cls._signal_hooks_registered
):
for signum in (signal.SIGTERM, signal.SIGINT):
root_cls._previous_signal_handlers[signum] = signal.getsignal(
signum
)
signal.signal(signum, root_cls._handle_parent_signal)
root_cls._signal_hooks_registered = True
def _register_active_server(self) -> None:
"""Track this server so parent-process exits still clean it up."""
RemoteVLLMServer._ensure_cleanup_hooks_registered()
with RemoteVLLMServer._active_servers_lock:
RemoteVLLMServer._active_servers.add(self)
def _unregister_active_server(self) -> None:
with RemoteVLLMServer._active_servers_lock:
RemoteVLLMServer._active_servers.discard(self)
@classmethod
def _shutdown_active_servers(cls) -> None:
"""Best-effort shutdown for all live RemoteVLLMServer instances."""
with cls._active_servers_lock:
servers = list(cls._active_servers)
for server in servers:
with contextlib.suppress(Exception):
server._shutdown()
@classmethod
def _handle_parent_signal(cls, signum, frame) -> None:
"""Clean up detached servers before letting the signal terminate pytest."""
cls._shutdown_active_servers()
previous_handler = cls._previous_signal_handlers.get(signum, signal.SIG_DFL)
if callable(previous_handler):
previous_handler(signum, frame)
elif previous_handler == signal.SIG_IGN:
return
elif signum == signal.SIGINT:
raise KeyboardInterrupt
else:
raise SystemExit(128 + signum)
def _terminate_process_tree(self) -> None:
"""Kill the server process tree without waiting for GPU memory release.
@@ -315,6 +385,9 @@ class RemoteVLLMServer:
if not servers:
return
for server in servers:
server._shutdown_complete = True
threads = [
threading.Thread(
target=s._terminate_process_tree,
@@ -339,7 +412,11 @@ class RemoteVLLMServer:
else s._pre_server_gpu_memory
),
)
earliest._wait_for_gpu_memory_release()
try:
earliest._wait_for_gpu_memory_release()
finally:
for server in servers:
server._unregister_active_server()
def _kill_process_group_survivors(
self, pgid: int | None, timeout: float = 15.0