From cb58bb9c1e38cd366910858873946c95be6328a9 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Wed, 19 Aug 2026 14:17:17 -0500 Subject: [PATCH] [CI] Harden RemoteVLLMServer GPU cleanup checks (#52282) Signed-off-by: Andreas Karatzas Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- tests/entrypoints/launchers/test_shutdown.py | 18 ++++++---- .../unit_tests/test_remote_vllm_server.py | 33 +++++++++++++++++++ tests/utils.py | 17 ++++++++-- 3 files changed, 59 insertions(+), 9 deletions(-) create mode 100644 tests/entrypoints/unit_tests/test_remote_vllm_server.py diff --git a/tests/entrypoints/launchers/test_shutdown.py b/tests/entrypoints/launchers/test_shutdown.py index e3be2919f5..643eff7594 100644 --- a/tests/entrypoints/launchers/test_shutdown.py +++ b/tests/entrypoints/launchers/test_shutdown.py @@ -272,7 +272,9 @@ async def test_wait_timeout_completes_requests(): @pytest.mark.asyncio @pytest.mark.parametrize("wait_for_engine_idle", [0.0, 2.0]) -async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float): +async def test_abort_timeout_exits_within_cleanup_grace( + wait_for_engine_idle: float, +): server_args = [ "--dtype", "bfloat16", @@ -303,19 +305,21 @@ async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float): # Wait for engine to become idle await asyncio.sleep(wait_for_engine_idle) - start_time = time.time() proc.send_signal(signal.SIGTERM) - # abort timeout (0) should stop the server promptly. + # A zero request timeout aborts requests immediately, but process + # teardown may still use the platform-specific resource cleanup grace. + termination_timeout = remote_server._get_process_termination_timeout() try: - proc.wait(timeout=4.0) + proc.wait(timeout=termination_timeout) except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=5) - pytest.fail("Process did not exit after SIGTERM with abort timeout") + pytest.fail( + "Process did not exit within the resource cleanup grace " + f"({termination_timeout}s)" + ) - exit_time = time.time() - start_time - assert exit_time < 4.1, f"Default shutdown took too long: {exit_time:.1f}s" assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}" await _assert_children_cleaned_up(child_pids) diff --git a/tests/entrypoints/unit_tests/test_remote_vllm_server.py b/tests/entrypoints/unit_tests/test_remote_vllm_server.py new file mode 100644 index 0000000000..7f22746c88 --- /dev/null +++ b/tests/entrypoints/unit_tests/test_remote_vllm_server.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from unittest.mock import Mock + +import pytest + +import tests.utils as test_utils +from tests.utils import RemoteOpenAIServer + + +def test_openai_server_shutdown_wait_covers_engine_cleanup( + monkeypatch: pytest.MonkeyPatch, +): + server = object.__new__(RemoteOpenAIServer) + server._request_shutdown_timeout = 0.0 + server.proc = Mock(pid=1234) + engine_timeout_args = [] + + def get_engine_timeout(request_timeout, process_timeout): + engine_timeout_args.append((request_timeout, process_timeout)) + return 60.0 + + monkeypatch.setattr( + test_utils, "get_engine_process_shutdown_timeout", get_engine_timeout + ) + monkeypatch.setattr(test_utils.os, "getpgid", lambda pid: pid) + monkeypatch.setattr(server, "_kill_process_group_survivors", Mock()) + + server._terminate_process_tree() + + assert engine_timeout_args == [(0.0, 0.0)] + server.proc.wait.assert_called_once_with(timeout=75.0) diff --git a/tests/utils.py b/tests/utils.py index 07601b74e4..71f9d4b669 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -63,6 +63,7 @@ from vllm.utils.network_utils import get_open_port from vllm.utils.torch_utils import ( set_random_seed, # noqa: F401 - re-exported for use in test files ) +from vllm.v1.engine.utils import get_engine_process_shutdown_timeout logger = init_logger(__name__) @@ -237,6 +238,9 @@ class RemoteVLLMServer: model_loader = get_model_loader(load_config) model_loader.download_model(model_config) + def _get_process_termination_timeout(self) -> float: + return 15.0 + def __init__( self, model: str, @@ -287,6 +291,7 @@ class RemoteVLLMServer: self.show_hidden_metrics = ( getattr(args, "show_hidden_metrics_for_version", None) is not None ) + self._request_shutdown_timeout = float(args.shutdown_timeout) with _temporarily_sanitized_pythonpath_env(): self._pre_download_model(model, args) @@ -416,7 +421,7 @@ class RemoteVLLMServer: print(f"[RemoteOpenAIServer] Sent SIGTERM to process {pid}") try: - self.proc.wait(timeout=15) + self.proc.wait(timeout=self._get_process_termination_timeout()) print(f"[RemoteOpenAIServer] Server {pid} terminated gracefully") except subprocess.TimeoutExpired: # Phase 2: SIGKILL the entire process group @@ -758,6 +763,14 @@ class RemoteVLLMServer: class RemoteOpenAIServer(RemoteVLLMServer): """Launches ``vllm serve`` for testing OpenAI-compatible endpoints.""" + def _get_process_termination_timeout(self) -> float: + engine_timeout = get_engine_process_shutdown_timeout( + self._request_shutdown_timeout, + self._request_shutdown_timeout, + ) + assert engine_timeout is not None + return engine_timeout + super()._get_process_termination_timeout() + def _create_cli_subcommand(self): return ServeSubcommand() @@ -889,7 +902,7 @@ class RemoteOpenAIServerCustom(RemoteOpenAIServer): self.proc.terminate() print(f"[RemoteOpenAIServerCustom] Sent SIGTERM to process {pid}") - self.proc.join(15) + self.proc.join(self._get_process_termination_timeout()) if self.proc.is_alive(): print( f"[RemoteOpenAIServerCustom] Server {pid} did not respond "