[CI] Harden RemoteVLLMServer GPU cleanup checks (#52282)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
480d4f0d15
commit
cb58bb9c1e
@@ -272,7 +272,9 @@ async def test_wait_timeout_completes_requests():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("wait_for_engine_idle", [0.0, 2.0])
|
||||
async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
|
||||
async def test_abort_timeout_exits_within_cleanup_grace(
|
||||
wait_for_engine_idle: float,
|
||||
):
|
||||
server_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
@@ -303,19 +305,21 @@ async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
|
||||
# Wait for engine to become idle
|
||||
await asyncio.sleep(wait_for_engine_idle)
|
||||
|
||||
start_time = time.time()
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
|
||||
# abort timeout (0) should stop the server promptly.
|
||||
# A zero request timeout aborts requests immediately, but process
|
||||
# teardown may still use the platform-specific resource cleanup grace.
|
||||
termination_timeout = remote_server._get_process_termination_timeout()
|
||||
try:
|
||||
proc.wait(timeout=4.0)
|
||||
proc.wait(timeout=termination_timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
pytest.fail("Process did not exit after SIGTERM with abort timeout")
|
||||
pytest.fail(
|
||||
"Process did not exit within the resource cleanup grace "
|
||||
f"({termination_timeout}s)"
|
||||
)
|
||||
|
||||
exit_time = time.time() - start_time
|
||||
assert exit_time < 4.1, f"Default shutdown took too long: {exit_time:.1f}s"
|
||||
assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}"
|
||||
|
||||
await _assert_children_cleaned_up(child_pids)
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
import tests.utils as test_utils
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
|
||||
def test_openai_server_shutdown_wait_covers_engine_cleanup(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
server = object.__new__(RemoteOpenAIServer)
|
||||
server._request_shutdown_timeout = 0.0
|
||||
server.proc = Mock(pid=1234)
|
||||
engine_timeout_args = []
|
||||
|
||||
def get_engine_timeout(request_timeout, process_timeout):
|
||||
engine_timeout_args.append((request_timeout, process_timeout))
|
||||
return 60.0
|
||||
|
||||
monkeypatch.setattr(
|
||||
test_utils, "get_engine_process_shutdown_timeout", get_engine_timeout
|
||||
)
|
||||
monkeypatch.setattr(test_utils.os, "getpgid", lambda pid: pid)
|
||||
monkeypatch.setattr(server, "_kill_process_group_survivors", Mock())
|
||||
|
||||
server._terminate_process_tree()
|
||||
|
||||
assert engine_timeout_args == [(0.0, 0.0)]
|
||||
server.proc.wait.assert_called_once_with(timeout=75.0)
|
||||
+15
-2
@@ -63,6 +63,7 @@ from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.torch_utils import (
|
||||
set_random_seed, # noqa: F401 - re-exported for use in test files
|
||||
)
|
||||
from vllm.v1.engine.utils import get_engine_process_shutdown_timeout
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -237,6 +238,9 @@ class RemoteVLLMServer:
|
||||
model_loader = get_model_loader(load_config)
|
||||
model_loader.download_model(model_config)
|
||||
|
||||
def _get_process_termination_timeout(self) -> float:
|
||||
return 15.0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
@@ -287,6 +291,7 @@ class RemoteVLLMServer:
|
||||
self.show_hidden_metrics = (
|
||||
getattr(args, "show_hidden_metrics_for_version", None) is not None
|
||||
)
|
||||
self._request_shutdown_timeout = float(args.shutdown_timeout)
|
||||
|
||||
with _temporarily_sanitized_pythonpath_env():
|
||||
self._pre_download_model(model, args)
|
||||
@@ -416,7 +421,7 @@ class RemoteVLLMServer:
|
||||
print(f"[RemoteOpenAIServer] Sent SIGTERM to process {pid}")
|
||||
|
||||
try:
|
||||
self.proc.wait(timeout=15)
|
||||
self.proc.wait(timeout=self._get_process_termination_timeout())
|
||||
print(f"[RemoteOpenAIServer] Server {pid} terminated gracefully")
|
||||
except subprocess.TimeoutExpired:
|
||||
# Phase 2: SIGKILL the entire process group
|
||||
@@ -758,6 +763,14 @@ class RemoteVLLMServer:
|
||||
class RemoteOpenAIServer(RemoteVLLMServer):
|
||||
"""Launches ``vllm serve`` for testing OpenAI-compatible endpoints."""
|
||||
|
||||
def _get_process_termination_timeout(self) -> float:
|
||||
engine_timeout = get_engine_process_shutdown_timeout(
|
||||
self._request_shutdown_timeout,
|
||||
self._request_shutdown_timeout,
|
||||
)
|
||||
assert engine_timeout is not None
|
||||
return engine_timeout + super()._get_process_termination_timeout()
|
||||
|
||||
def _create_cli_subcommand(self):
|
||||
return ServeSubcommand()
|
||||
|
||||
@@ -889,7 +902,7 @@ class RemoteOpenAIServerCustom(RemoteOpenAIServer):
|
||||
self.proc.terminate()
|
||||
print(f"[RemoteOpenAIServerCustom] Sent SIGTERM to process {pid}")
|
||||
|
||||
self.proc.join(15)
|
||||
self.proc.join(self._get_process_termination_timeout())
|
||||
if self.proc.is_alive():
|
||||
print(
|
||||
f"[RemoteOpenAIServerCustom] Server {pid} did not respond "
|
||||
|
||||
Reference in New Issue
Block a user