diff --git a/tests/utils.py b/tests/utils.py index e4b6a6ff6e..d2d07937c3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio +import atexit import contextlib import copy import functools @@ -134,6 +135,11 @@ class RemoteVLLMServer: """ DUMMY_API_KEY = "token-abc123" # vLLM's OpenAI server does not need API key + _active_servers: set["RemoteVLLMServer"] = set() + _active_servers_lock = threading.RLock() + _cleanup_hooks_registered = False + _signal_hooks_registered = False + _previous_signal_handlers: dict[int, Any] = {} proc: subprocess.Popen def _create_cli_subcommand(self): @@ -209,6 +215,7 @@ class RemoteVLLMServer: ) self._pre_download_model(model, args) + self._shutdown_complete = False # Record GPU memory before server start so we know what # "released" looks like. @@ -221,6 +228,7 @@ class RemoteVLLMServer: ) self._start_server(model, vllm_serve_args, env_dict) + self._register_active_server() max_wait_seconds = max_wait_seconds or 480 try: self._wait_for_server(url=self.url_for("health"), timeout=max_wait_seconds) @@ -246,8 +254,70 @@ class RemoteVLLMServer: (when the server fails to start). Must be safe to call even if the process is already dead. """ - self._terminate_process_tree() - self._wait_for_gpu_memory_release() + if self._shutdown_complete: + return + + self._shutdown_complete = True + try: + self._terminate_process_tree() + self._wait_for_gpu_memory_release() + finally: + self._unregister_active_server() + + @classmethod + def _ensure_cleanup_hooks_registered(cls) -> None: + """Register process-exit cleanup for detached server subprocesses.""" + root_cls = RemoteVLLMServer + with root_cls._active_servers_lock: + if not root_cls._cleanup_hooks_registered: + atexit.register(root_cls._shutdown_active_servers) + root_cls._cleanup_hooks_registered = True + + if ( + threading.current_thread() is threading.main_thread() + and not root_cls._signal_hooks_registered + ): + for signum in (signal.SIGTERM, signal.SIGINT): + root_cls._previous_signal_handlers[signum] = signal.getsignal( + signum + ) + signal.signal(signum, root_cls._handle_parent_signal) + root_cls._signal_hooks_registered = True + + def _register_active_server(self) -> None: + """Track this server so parent-process exits still clean it up.""" + RemoteVLLMServer._ensure_cleanup_hooks_registered() + with RemoteVLLMServer._active_servers_lock: + RemoteVLLMServer._active_servers.add(self) + + def _unregister_active_server(self) -> None: + with RemoteVLLMServer._active_servers_lock: + RemoteVLLMServer._active_servers.discard(self) + + @classmethod + def _shutdown_active_servers(cls) -> None: + """Best-effort shutdown for all live RemoteVLLMServer instances.""" + with cls._active_servers_lock: + servers = list(cls._active_servers) + + for server in servers: + with contextlib.suppress(Exception): + server._shutdown() + + @classmethod + def _handle_parent_signal(cls, signum, frame) -> None: + """Clean up detached servers before letting the signal terminate pytest.""" + cls._shutdown_active_servers() + + previous_handler = cls._previous_signal_handlers.get(signum, signal.SIG_DFL) + if callable(previous_handler): + previous_handler(signum, frame) + elif previous_handler == signal.SIG_IGN: + return + elif signum == signal.SIGINT: + raise KeyboardInterrupt + else: + raise SystemExit(128 + signum) def _terminate_process_tree(self) -> None: """Kill the server process tree without waiting for GPU memory release. @@ -315,6 +385,9 @@ class RemoteVLLMServer: if not servers: return + for server in servers: + server._shutdown_complete = True + threads = [ threading.Thread( target=s._terminate_process_tree, @@ -339,7 +412,11 @@ class RemoteVLLMServer: else s._pre_server_gpu_memory ), ) - earliest._wait_for_gpu_memory_release() + try: + earliest._wait_for_gpu_memory_release() + finally: + for server in servers: + server._unregister_active_server() def _kill_process_group_survivors( self, pgid: int | None, timeout: float = 15.0