[CI] Harden RemoteVLLMServer GPU cleanup checks (#52282)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
480d4f0d15
commit
cb58bb9c1e
@@ -272,7 +272,9 @@ async def test_wait_timeout_completes_requests():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("wait_for_engine_idle", [0.0, 2.0])
|
||||
async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
|
||||
async def test_abort_timeout_exits_within_cleanup_grace(
|
||||
wait_for_engine_idle: float,
|
||||
):
|
||||
server_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
@@ -303,19 +305,21 @@ async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
|
||||
# Wait for engine to become idle
|
||||
await asyncio.sleep(wait_for_engine_idle)
|
||||
|
||||
start_time = time.time()
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
|
||||
# abort timeout (0) should stop the server promptly.
|
||||
# A zero request timeout aborts requests immediately, but process
|
||||
# teardown may still use the platform-specific resource cleanup grace.
|
||||
termination_timeout = remote_server._get_process_termination_timeout()
|
||||
try:
|
||||
proc.wait(timeout=4.0)
|
||||
proc.wait(timeout=termination_timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
pytest.fail("Process did not exit after SIGTERM with abort timeout")
|
||||
pytest.fail(
|
||||
"Process did not exit within the resource cleanup grace "
|
||||
f"({termination_timeout}s)"
|
||||
)
|
||||
|
||||
exit_time = time.time() - start_time
|
||||
assert exit_time < 4.1, f"Default shutdown took too long: {exit_time:.1f}s"
|
||||
assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}"
|
||||
|
||||
await _assert_children_cleaned_up(child_pids)
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
import tests.utils as test_utils
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
|
||||
def test_openai_server_shutdown_wait_covers_engine_cleanup(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
server = object.__new__(RemoteOpenAIServer)
|
||||
server._request_shutdown_timeout = 0.0
|
||||
server.proc = Mock(pid=1234)
|
||||
engine_timeout_args = []
|
||||
|
||||
def get_engine_timeout(request_timeout, process_timeout):
|
||||
engine_timeout_args.append((request_timeout, process_timeout))
|
||||
return 60.0
|
||||
|
||||
monkeypatch.setattr(
|
||||
test_utils, "get_engine_process_shutdown_timeout", get_engine_timeout
|
||||
)
|
||||
monkeypatch.setattr(test_utils.os, "getpgid", lambda pid: pid)
|
||||
monkeypatch.setattr(server, "_kill_process_group_survivors", Mock())
|
||||
|
||||
server._terminate_process_tree()
|
||||
|
||||
assert engine_timeout_args == [(0.0, 0.0)]
|
||||
server.proc.wait.assert_called_once_with(timeout=75.0)
|
||||
Reference in New Issue
Block a user