[Ci][BugFix] Fix slow DP tests due to bad teardown logic (#41166)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-04-29 19:31:00 -04:00
committed by GitHub
parent 296741d025
commit 18599bfdf2
4 changed files with 81 additions and 30 deletions
+55 -6
View File
@@ -14,9 +14,10 @@ import signal
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
import threading
import time import time
import warnings import warnings
from collections.abc import Callable, Iterable from collections.abc import Callable, Iterable, Sequence
from contextlib import ExitStack, contextmanager, suppress from contextlib import ExitStack, contextmanager, suppress
from multiprocessing import Process from multiprocessing import Process
from pathlib import Path from pathlib import Path
@@ -62,8 +63,6 @@ from vllm.utils.torch_utils import (
FP8_DTYPE = current_platform.fp8_dtype() FP8_DTYPE = current_platform.fp8_dtype()
if current_platform.is_rocm(): if current_platform.is_rocm():
import threading
from amdsmi import ( from amdsmi import (
amdsmi_get_gpu_vram_usage, amdsmi_get_gpu_vram_usage,
amdsmi_get_processor_handles, amdsmi_get_processor_handles,
@@ -247,6 +246,16 @@ class RemoteVLLMServer:
(when the server fails to start). Must be safe to call even if (when the server fails to start). Must be safe to call even if
the process is already dead. the process is already dead.
""" """
self._terminate_process_tree()
self._wait_for_gpu_memory_release()
def _terminate_process_tree(self) -> None:
"""Kill the server process tree without waiting for GPU memory release.
Split out from ``_shutdown`` so that ``shutdown_many`` can run this
phase in parallel for sibling servers and then wait for GPU memory
release once at the end.
"""
pid = self.proc.pid pid = self.proc.pid
# Get the process group ID. Because we used # Get the process group ID. Because we used
@@ -288,9 +297,49 @@ class RemoteVLLMServer:
# prevent VRAM from being reclaimed by the driver. # prevent VRAM from being reclaimed by the driver.
self._kill_process_group_survivors(pgid) self._kill_process_group_survivors(pgid)
# Wait for GPU memory to actually be freed, not just @classmethod
# "stabilized at whatever level it's at". def shutdown_many(cls, servers: Sequence["RemoteVLLMServer"]) -> None:
self._wait_for_gpu_memory_release() """Shut down multiple sibling servers and wait for GPU memory once.
Test fixtures that hold several ``RemoteVLLMServer`` instances at
once must NOT shut them down by calling each server's ``__exit__``
sequentially: every server measures total GPU memory across all
visible devices in ``_wait_for_gpu_memory_release``, so the first
server's wait blocks the full timeout because later sibling
servers are still holding GPU memory.
Instead, this method terminates every server's process tree in
parallel, then runs the GPU-memory-release wait once against the
earliest recorded baseline (memory before any server started).
"""
if not servers:
return
threads = [
threading.Thread(
target=s._terminate_process_tree,
name=f"shutdown-{s.proc.pid}",
daemon=True,
)
for s in servers
]
for t in threads:
t.start()
for t in threads:
t.join()
# Use the smallest pre-server baseline so the wait targets memory
# usage before *any* of these sibling servers started, not after
# earlier siblings had already allocated.
earliest = min(
servers,
key=lambda s: (
float("inf")
if s._pre_server_gpu_memory is None
else s._pre_server_gpu_memory
),
)
earliest._wait_for_gpu_memory_release()
def _kill_process_group_survivors( def _kill_process_group_survivors(
self, pgid: int | None, timeout: float = 15.0 self, pgid: int | None, timeout: float = 15.0
+6 -5
View File
@@ -111,11 +111,12 @@ class ExternalLBServerManager:
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
"""Stop all server instances.""" """Stop all server instances."""
while self.servers: servers = [s for s, _ in self.servers]
try: self.servers.clear()
self.servers.pop()[0].__exit__(exc_type, exc_val, exc_tb) try:
except Exception as e: RemoteOpenAIServer.shutdown_many(servers)
print(f"Error stopping server: {e}") except Exception as e:
print(f"Error stopping servers: {e}")
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
+6 -5
View File
@@ -134,11 +134,12 @@ class HybridLBServerManager:
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
"""Stop all server instances.""" """Stop all server instances."""
while self.servers: servers = [s for s, _ in self.servers]
try: self.servers.clear()
self.servers.pop()[0].__exit__(exc_type, exc_val, exc_tb) try:
except Exception as e: RemoteOpenAIServer.shutdown_many(servers)
print(f"Error stopping server: {e}") except Exception as e:
print(f"Error stopping servers: {e}")
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
+14 -14
View File
@@ -228,13 +228,13 @@ class MultinodeInternalLBServerManager:
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
"""Stop all server instances.""" """Stop all server instances."""
while self.servers: servers = [entry[0] for entry in self.servers if entry is not None]
if server := self.servers.pop(): self.servers.clear()
try: try:
server[0].__exit__(exc_type, exc_val, exc_tb) RemoteOpenAIServer.shutdown_many(servers)
except Exception as e: except Exception as e:
print(f"Error stopping server: {e}") print(f"Error stopping servers: {e}")
traceback.print_exc() traceback.print_exc()
class APIOnlyServerManager: class APIOnlyServerManager:
@@ -370,13 +370,13 @@ class APIOnlyServerManager:
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
"""Stop both server instances.""" """Stop both server instances."""
while self.servers: servers = [entry[0] for entry in self.servers if entry is not None]
if server := self.servers.pop(): self.servers.clear()
try: try:
server[0].__exit__(exc_type, exc_val, exc_tb) RemoteOpenAIServer.shutdown_many(servers)
except Exception as e: except Exception as e:
print(f"Error stopping server: {e}") print(f"Error stopping servers: {e}")
traceback.print_exc() traceback.print_exc()
@pytest.fixture(scope="module") @pytest.fixture(scope="module")