[Ci][BugFix] Fix slow DP tests due to bad teardown logic (#41166)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
+55
-6
@@ -14,9 +14,10 @@ import signal
|
|||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import threading
|
||||||
import time
|
import time
|
||||||
import warnings
|
import warnings
|
||||||
from collections.abc import Callable, Iterable
|
from collections.abc import Callable, Iterable, Sequence
|
||||||
from contextlib import ExitStack, contextmanager, suppress
|
from contextlib import ExitStack, contextmanager, suppress
|
||||||
from multiprocessing import Process
|
from multiprocessing import Process
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -62,8 +63,6 @@ from vllm.utils.torch_utils import (
|
|||||||
FP8_DTYPE = current_platform.fp8_dtype()
|
FP8_DTYPE = current_platform.fp8_dtype()
|
||||||
|
|
||||||
if current_platform.is_rocm():
|
if current_platform.is_rocm():
|
||||||
import threading
|
|
||||||
|
|
||||||
from amdsmi import (
|
from amdsmi import (
|
||||||
amdsmi_get_gpu_vram_usage,
|
amdsmi_get_gpu_vram_usage,
|
||||||
amdsmi_get_processor_handles,
|
amdsmi_get_processor_handles,
|
||||||
@@ -247,6 +246,16 @@ class RemoteVLLMServer:
|
|||||||
(when the server fails to start). Must be safe to call even if
|
(when the server fails to start). Must be safe to call even if
|
||||||
the process is already dead.
|
the process is already dead.
|
||||||
"""
|
"""
|
||||||
|
self._terminate_process_tree()
|
||||||
|
self._wait_for_gpu_memory_release()
|
||||||
|
|
||||||
|
def _terminate_process_tree(self) -> None:
|
||||||
|
"""Kill the server process tree without waiting for GPU memory release.
|
||||||
|
|
||||||
|
Split out from ``_shutdown`` so that ``shutdown_many`` can run this
|
||||||
|
phase in parallel for sibling servers and then wait for GPU memory
|
||||||
|
release once at the end.
|
||||||
|
"""
|
||||||
pid = self.proc.pid
|
pid = self.proc.pid
|
||||||
|
|
||||||
# Get the process group ID. Because we used
|
# Get the process group ID. Because we used
|
||||||
@@ -288,9 +297,49 @@ class RemoteVLLMServer:
|
|||||||
# prevent VRAM from being reclaimed by the driver.
|
# prevent VRAM from being reclaimed by the driver.
|
||||||
self._kill_process_group_survivors(pgid)
|
self._kill_process_group_survivors(pgid)
|
||||||
|
|
||||||
# Wait for GPU memory to actually be freed, not just
|
@classmethod
|
||||||
# "stabilized at whatever level it's at".
|
def shutdown_many(cls, servers: Sequence["RemoteVLLMServer"]) -> None:
|
||||||
self._wait_for_gpu_memory_release()
|
"""Shut down multiple sibling servers and wait for GPU memory once.
|
||||||
|
|
||||||
|
Test fixtures that hold several ``RemoteVLLMServer`` instances at
|
||||||
|
once must NOT shut them down by calling each server's ``__exit__``
|
||||||
|
sequentially: every server measures total GPU memory across all
|
||||||
|
visible devices in ``_wait_for_gpu_memory_release``, so the first
|
||||||
|
server's wait blocks the full timeout because later sibling
|
||||||
|
servers are still holding GPU memory.
|
||||||
|
|
||||||
|
Instead, this method terminates every server's process tree in
|
||||||
|
parallel, then runs the GPU-memory-release wait once against the
|
||||||
|
earliest recorded baseline (memory before any server started).
|
||||||
|
"""
|
||||||
|
if not servers:
|
||||||
|
return
|
||||||
|
|
||||||
|
threads = [
|
||||||
|
threading.Thread(
|
||||||
|
target=s._terminate_process_tree,
|
||||||
|
name=f"shutdown-{s.proc.pid}",
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
|
for s in servers
|
||||||
|
]
|
||||||
|
for t in threads:
|
||||||
|
t.start()
|
||||||
|
for t in threads:
|
||||||
|
t.join()
|
||||||
|
|
||||||
|
# Use the smallest pre-server baseline so the wait targets memory
|
||||||
|
# usage before *any* of these sibling servers started, not after
|
||||||
|
# earlier siblings had already allocated.
|
||||||
|
earliest = min(
|
||||||
|
servers,
|
||||||
|
key=lambda s: (
|
||||||
|
float("inf")
|
||||||
|
if s._pre_server_gpu_memory is None
|
||||||
|
else s._pre_server_gpu_memory
|
||||||
|
),
|
||||||
|
)
|
||||||
|
earliest._wait_for_gpu_memory_release()
|
||||||
|
|
||||||
def _kill_process_group_survivors(
|
def _kill_process_group_survivors(
|
||||||
self, pgid: int | None, timeout: float = 15.0
|
self, pgid: int | None, timeout: float = 15.0
|
||||||
|
|||||||
@@ -111,11 +111,12 @@ class ExternalLBServerManager:
|
|||||||
|
|
||||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
"""Stop all server instances."""
|
"""Stop all server instances."""
|
||||||
while self.servers:
|
servers = [s for s, _ in self.servers]
|
||||||
try:
|
self.servers.clear()
|
||||||
self.servers.pop()[0].__exit__(exc_type, exc_val, exc_tb)
|
try:
|
||||||
except Exception as e:
|
RemoteOpenAIServer.shutdown_many(servers)
|
||||||
print(f"Error stopping server: {e}")
|
except Exception as e:
|
||||||
|
print(f"Error stopping servers: {e}")
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
|
|||||||
@@ -134,11 +134,12 @@ class HybridLBServerManager:
|
|||||||
|
|
||||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
"""Stop all server instances."""
|
"""Stop all server instances."""
|
||||||
while self.servers:
|
servers = [s for s, _ in self.servers]
|
||||||
try:
|
self.servers.clear()
|
||||||
self.servers.pop()[0].__exit__(exc_type, exc_val, exc_tb)
|
try:
|
||||||
except Exception as e:
|
RemoteOpenAIServer.shutdown_many(servers)
|
||||||
print(f"Error stopping server: {e}")
|
except Exception as e:
|
||||||
|
print(f"Error stopping servers: {e}")
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
|
|||||||
@@ -228,13 +228,13 @@ class MultinodeInternalLBServerManager:
|
|||||||
|
|
||||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
"""Stop all server instances."""
|
"""Stop all server instances."""
|
||||||
while self.servers:
|
servers = [entry[0] for entry in self.servers if entry is not None]
|
||||||
if server := self.servers.pop():
|
self.servers.clear()
|
||||||
try:
|
try:
|
||||||
server[0].__exit__(exc_type, exc_val, exc_tb)
|
RemoteOpenAIServer.shutdown_many(servers)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error stopping server: {e}")
|
print(f"Error stopping servers: {e}")
|
||||||
traceback.print_exc()
|
traceback.print_exc()
|
||||||
|
|
||||||
|
|
||||||
class APIOnlyServerManager:
|
class APIOnlyServerManager:
|
||||||
@@ -370,13 +370,13 @@ class APIOnlyServerManager:
|
|||||||
|
|
||||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
"""Stop both server instances."""
|
"""Stop both server instances."""
|
||||||
while self.servers:
|
servers = [entry[0] for entry in self.servers if entry is not None]
|
||||||
if server := self.servers.pop():
|
self.servers.clear()
|
||||||
try:
|
try:
|
||||||
server[0].__exit__(exc_type, exc_val, exc_tb)
|
RemoteOpenAIServer.shutdown_many(servers)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error stopping server: {e}")
|
print(f"Error stopping servers: {e}")
|
||||||
traceback.print_exc()
|
traceback.print_exc()
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
|
|||||||
Reference in New Issue
Block a user