[CI] Harden RemoteVLLMServer GPU cleanup checks (#52282)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-08-19 14:17:17 -05:00
committed by GitHub
co-authored by OpenAI Codex
parent 480d4f0d15
commit cb58bb9c1e
3 changed files with 59 additions and 9 deletions
+11 -7
View File
@@ -272,7 +272,9 @@ async def test_wait_timeout_completes_requests():
@pytest.mark.asyncio
@pytest.mark.parametrize("wait_for_engine_idle", [0.0, 2.0])
async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
async def test_abort_timeout_exits_within_cleanup_grace(
wait_for_engine_idle: float,
):
server_args = [
"--dtype",
"bfloat16",
@@ -303,19 +305,21 @@ async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
# Wait for engine to become idle
await asyncio.sleep(wait_for_engine_idle)
start_time = time.time()
proc.send_signal(signal.SIGTERM)
# abort timeout (0) should stop the server promptly.
# A zero request timeout aborts requests immediately, but process
# teardown may still use the platform-specific resource cleanup grace.
termination_timeout = remote_server._get_process_termination_timeout()
try:
proc.wait(timeout=4.0)
proc.wait(timeout=termination_timeout)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait(timeout=5)
pytest.fail("Process did not exit after SIGTERM with abort timeout")
pytest.fail(
"Process did not exit within the resource cleanup grace "
f"({termination_timeout}s)"
)
exit_time = time.time() - start_time
assert exit_time < 4.1, f"Default shutdown took too long: {exit_time:.1f}s"
assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}"
await _assert_children_cleaned_up(child_pids)
@@ -0,0 +1,33 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import Mock
import pytest
import tests.utils as test_utils
from tests.utils import RemoteOpenAIServer
def test_openai_server_shutdown_wait_covers_engine_cleanup(
monkeypatch: pytest.MonkeyPatch,
):
server = object.__new__(RemoteOpenAIServer)
server._request_shutdown_timeout = 0.0
server.proc = Mock(pid=1234)
engine_timeout_args = []
def get_engine_timeout(request_timeout, process_timeout):
engine_timeout_args.append((request_timeout, process_timeout))
return 60.0
monkeypatch.setattr(
test_utils, "get_engine_process_shutdown_timeout", get_engine_timeout
)
monkeypatch.setattr(test_utils.os, "getpgid", lambda pid: pid)
monkeypatch.setattr(server, "_kill_process_group_survivors", Mock())
server._terminate_process_tree()
assert engine_timeout_args == [(0.0, 0.0)]
server.proc.wait.assert_called_once_with(timeout=75.0)