[CI] Harden RemoteVLLMServer GPU cleanup checks (#52282)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-08-19 14:17:17 -05:00
committed by GitHub
co-authored by OpenAI Codex
parent 480d4f0d15
commit cb58bb9c1e
3 changed files with 59 additions and 9 deletions
+15 -2
View File
@@ -63,6 +63,7 @@ from vllm.utils.network_utils import get_open_port
from vllm.utils.torch_utils import (
set_random_seed, # noqa: F401 - re-exported for use in test files
)
from vllm.v1.engine.utils import get_engine_process_shutdown_timeout
logger = init_logger(__name__)
@@ -237,6 +238,9 @@ class RemoteVLLMServer:
model_loader = get_model_loader(load_config)
model_loader.download_model(model_config)
def _get_process_termination_timeout(self) -> float:
return 15.0
def __init__(
self,
model: str,
@@ -287,6 +291,7 @@ class RemoteVLLMServer:
self.show_hidden_metrics = (
getattr(args, "show_hidden_metrics_for_version", None) is not None
)
self._request_shutdown_timeout = float(args.shutdown_timeout)
with _temporarily_sanitized_pythonpath_env():
self._pre_download_model(model, args)
@@ -416,7 +421,7 @@ class RemoteVLLMServer:
print(f"[RemoteOpenAIServer] Sent SIGTERM to process {pid}")
try:
self.proc.wait(timeout=15)
self.proc.wait(timeout=self._get_process_termination_timeout())
print(f"[RemoteOpenAIServer] Server {pid} terminated gracefully")
except subprocess.TimeoutExpired:
# Phase 2: SIGKILL the entire process group
@@ -758,6 +763,14 @@ class RemoteVLLMServer:
class RemoteOpenAIServer(RemoteVLLMServer):
"""Launches ``vllm serve`` for testing OpenAI-compatible endpoints."""
def _get_process_termination_timeout(self) -> float:
engine_timeout = get_engine_process_shutdown_timeout(
self._request_shutdown_timeout,
self._request_shutdown_timeout,
)
assert engine_timeout is not None
return engine_timeout + super()._get_process_termination_timeout()
def _create_cli_subcommand(self):
return ServeSubcommand()
@@ -889,7 +902,7 @@ class RemoteOpenAIServerCustom(RemoteOpenAIServer):
self.proc.terminate()
print(f"[RemoteOpenAIServerCustom] Sent SIGTERM to process {pid}")
self.proc.join(15)
self.proc.join(self._get_process_termination_timeout())
if self.proc.is_alive():
print(
f"[RemoteOpenAIServerCustom] Server {pid} did not respond "