[CI] Harden RemoteVLLMServer GPU cleanup checks (#52282)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-08-19 14:17:17 -05:00
committed by GitHub
co-authored by OpenAI Codex
parent 480d4f0d15
commit cb58bb9c1e
3 changed files with 59 additions and 9 deletions
+11 -7
View File
@@ -272,7 +272,9 @@ async def test_wait_timeout_completes_requests():
@pytest.mark.asyncio
@pytest.mark.parametrize("wait_for_engine_idle", [0.0, 2.0])
async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
async def test_abort_timeout_exits_within_cleanup_grace(
wait_for_engine_idle: float,
):
server_args = [
"--dtype",
"bfloat16",
@@ -303,19 +305,21 @@ async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
# Wait for engine to become idle
await asyncio.sleep(wait_for_engine_idle)
start_time = time.time()
proc.send_signal(signal.SIGTERM)
# abort timeout (0) should stop the server promptly.
# A zero request timeout aborts requests immediately, but process
# teardown may still use the platform-specific resource cleanup grace.
termination_timeout = remote_server._get_process_termination_timeout()
try:
proc.wait(timeout=4.0)
proc.wait(timeout=termination_timeout)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait(timeout=5)
pytest.fail("Process did not exit after SIGTERM with abort timeout")
pytest.fail(
"Process did not exit within the resource cleanup grace "
f"({termination_timeout}s)"
)
exit_time = time.time() - start_time
assert exit_time < 4.1, f"Default shutdown took too long: {exit_time:.1f}s"
assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}"
await _assert_children_cleaned_up(child_pids)
@@ -0,0 +1,33 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import Mock
import pytest
import tests.utils as test_utils
from tests.utils import RemoteOpenAIServer
def test_openai_server_shutdown_wait_covers_engine_cleanup(
monkeypatch: pytest.MonkeyPatch,
):
server = object.__new__(RemoteOpenAIServer)
server._request_shutdown_timeout = 0.0
server.proc = Mock(pid=1234)
engine_timeout_args = []
def get_engine_timeout(request_timeout, process_timeout):
engine_timeout_args.append((request_timeout, process_timeout))
return 60.0
monkeypatch.setattr(
test_utils, "get_engine_process_shutdown_timeout", get_engine_timeout
)
monkeypatch.setattr(test_utils.os, "getpgid", lambda pid: pid)
monkeypatch.setattr(server, "_kill_process_group_survivors", Mock())
server._terminate_process_tree()
assert engine_timeout_args == [(0.0, 0.0)]
server.proc.wait.assert_called_once_with(timeout=75.0)
+15 -2
View File
@@ -63,6 +63,7 @@ from vllm.utils.network_utils import get_open_port
from vllm.utils.torch_utils import (
set_random_seed, # noqa: F401 - re-exported for use in test files
)
from vllm.v1.engine.utils import get_engine_process_shutdown_timeout
logger = init_logger(__name__)
@@ -237,6 +238,9 @@ class RemoteVLLMServer:
model_loader = get_model_loader(load_config)
model_loader.download_model(model_config)
def _get_process_termination_timeout(self) -> float:
return 15.0
def __init__(
self,
model: str,
@@ -287,6 +291,7 @@ class RemoteVLLMServer:
self.show_hidden_metrics = (
getattr(args, "show_hidden_metrics_for_version", None) is not None
)
self._request_shutdown_timeout = float(args.shutdown_timeout)
with _temporarily_sanitized_pythonpath_env():
self._pre_download_model(model, args)
@@ -416,7 +421,7 @@ class RemoteVLLMServer:
print(f"[RemoteOpenAIServer] Sent SIGTERM to process {pid}")
try:
self.proc.wait(timeout=15)
self.proc.wait(timeout=self._get_process_termination_timeout())
print(f"[RemoteOpenAIServer] Server {pid} terminated gracefully")
except subprocess.TimeoutExpired:
# Phase 2: SIGKILL the entire process group
@@ -758,6 +763,14 @@ class RemoteVLLMServer:
class RemoteOpenAIServer(RemoteVLLMServer):
"""Launches ``vllm serve`` for testing OpenAI-compatible endpoints."""
def _get_process_termination_timeout(self) -> float:
engine_timeout = get_engine_process_shutdown_timeout(
self._request_shutdown_timeout,
self._request_shutdown_timeout,
)
assert engine_timeout is not None
return engine_timeout + super()._get_process_termination_timeout()
def _create_cli_subcommand(self):
return ServeSubcommand()
@@ -889,7 +902,7 @@ class RemoteOpenAIServerCustom(RemoteOpenAIServer):
self.proc.terminate()
print(f"[RemoteOpenAIServerCustom] Sent SIGTERM to process {pid}")
self.proc.join(15)
self.proc.join(self._get_process_termination_timeout())
if self.proc.is_alive():
print(
f"[RemoteOpenAIServerCustom] Server {pid} did not respond "