[CI][MCP][Harmony] Heavy refactoring Harmony & MCP response tests and stabilizing with deterministic test infrastructure (#33949)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-02-20 20:03:32 -08:00
committed by GitHub
parent 5719a4e4e6
commit 991d6bff38
10 changed files with 1186 additions and 885 deletions
+123 -30
View File
@@ -128,6 +128,9 @@ class RemoteOpenAIServer:
env=env,
stdout=sys.stdout,
stderr=sys.stderr,
# Create a dedicated process group so we can kill
# the entire tree (parent + EngineCore + workers) at once.
start_new_session=True,
)
def __init__(
@@ -189,6 +192,15 @@ class RemoteOpenAIServer:
model_loader = get_model_loader(load_config)
model_loader.download_model(model_config)
# Record GPU memory before server start so we know what
# "released" looks like.
self._pre_server_gpu_memory = self._get_gpu_memory_used()
if self._pre_server_gpu_memory is not None:
pre_gb = self._pre_server_gpu_memory / 1e9
print(
f"[RemoteOpenAIServer] GPU memory before server start: {pre_gb:.2f} GB"
)
self._start_server(model, vllm_serve_args, env_dict)
max_wait_seconds = max_wait_seconds or 360
self._wait_for_server(url=self.url_for("health"), timeout=max_wait_seconds)
@@ -198,27 +210,69 @@ class RemoteOpenAIServer:
def __exit__(self, exc_type, exc_value, traceback):
pid = self.proc.pid
# Graceful shutdown
self.proc.terminate()
# Get the process group ID. Because we used
# start_new_session=True the pgid equals the server's pid.
try:
pgid = os.getpgid(pid)
except (ProcessLookupError, OSError):
pgid = None
# Phase 1: graceful SIGTERM to the entire process group
if pgid is not None:
with contextlib.suppress(ProcessLookupError, OSError):
os.killpg(pgid, signal.SIGTERM)
print(f"[RemoteOpenAIServer] Sent SIGTERM to process group {pgid}")
else:
self.proc.terminate()
try:
self.proc.wait(timeout=15)
print(f"[RemoteOpenAIServer] Server {pid} terminated gracefully")
except subprocess.TimeoutExpired:
# Phase 2: SIGKILL the entire process group
print(
f"[RemoteOpenAIServer] Server {pid} did not respond "
"to SIGTERM, sending SIGKILL"
"to SIGTERM, sending SIGKILL to process group"
)
self.proc.kill()
if pgid is not None:
with contextlib.suppress(ProcessLookupError, OSError):
os.killpg(pgid, signal.SIGKILL)
else:
self.proc.kill()
try:
self.proc.wait(timeout=5)
self.proc.wait(timeout=10)
print(f"[RemoteOpenAIServer] Server {pid} killed")
except subprocess.TimeoutExpired as err:
raise RuntimeError(
f"[RemoteOpenAIServer] Failed to kill server process {pid}"
) from err
# Wait for GPU memory to be released
except subprocess.TimeoutExpired:
# Phase 3: last resort - find and kill any orphaned children
self._kill_orphaned_children(pid)
# Wait for GPU memory to actually be *freed*, not just
# "stabilized at whatever level it's at".
self._wait_for_gpu_memory_release()
def _kill_orphaned_children(self, parent_pid: int) -> None:
"""Best-effort cleanup of any lingering child processes."""
try:
import psutil
parent = psutil.Process(parent_pid)
children = parent.children(recursive=True)
for child in children:
print(
f"[RemoteOpenAIServer] Killing orphaned child "
f"pid={child.pid} name={child.name()}"
)
child.kill()
psutil.wait_procs(children, timeout=5)
except Exception as e:
# psutil may not be installed, or processes already gone
print(f"[RemoteOpenAIServer] Orphan cleanup failed: {e}")
# Fallback: try to kill by pgid one more time
with contextlib.suppress(ProcessLookupError, OSError):
os.killpg(parent_pid, signal.SIGKILL)
def _get_gpu_memory_used(self) -> float | None:
"""Get total GPU memory used across all visible devices in bytes."""
try:
@@ -244,10 +298,26 @@ class RemoteOpenAIServer:
return None
return None
def _wait_for_gpu_memory_release(self, timeout: float = 30.0):
"""Poll GPU memory until it stabilizes, indicating cleanup is complete."""
def _wait_for_gpu_memory_release(self, timeout: float = 60.0):
"""Wait for GPU memory to drop back toward pre-server levels.
Two-phase strategy:
1. Try to wait for memory to return close to pre-server baseline.
2. If that doesn't happen, fall back to waiting for stabilization
and log a warning (the next server might still OOM).
"""
baseline = self._pre_server_gpu_memory
if baseline is None:
# Can't query GPU memory - nothing to do
return
# Allow up to 2 GiB overhead above baseline for driver/context state
# that may persist between server instances.
headroom_bytes = 2 * 1024 * 1024 * 1024
target = baseline + headroom_bytes
start = time.time()
prev_used: float | None = None
last_used: float | None = None
stable_count = 0
while time.time() - start < timeout:
@@ -256,26 +326,49 @@ class RemoteOpenAIServer:
if used is None:
return # Can't query, assume ok
if prev_used is not None and abs(used - prev_used) < 100 * 1024 * 1024:
stable_count += 1
if stable_count >= 3:
used_gb = used / 1e9
print(
f"[RemoteOpenAIServer] GPU memory stabilized "
f"at {used_gb:.2f} GB"
)
return
else:
stable_count = 0
used_gb = used / 1e9
target_gb = target / 1e9
elapsed = time.time() - start
prev_used = used
time.sleep(0.1)
# Phase 1: memory dropped to near baseline - we're done.
if used <= target:
print(
f"[RemoteOpenAIServer] GPU memory released to "
f"{used_gb:.2f} GB (target: {target_gb:.2f} GB) "
f"in {elapsed:.1f}s"
)
return
last_reading = prev_used / 1e9 if prev_used is not None else 0.0
# Phase 2 (after 40s): fall back to stabilization check.
# This handles cases where another process is using GPU memory
# and we'll never reach baseline.
if elapsed > 40.0 and last_used is not None:
delta = abs(used - last_used)
if delta < 200 * 1024 * 1024: # 200 MB
stable_count += 1
if stable_count >= 3:
print(
f"[RemoteOpenAIServer] WARNING: GPU memory "
f"stabilized at {used_gb:.2f} GB "
f"(target was {target_gb:.2f} GB). "
f"Proceeding - next server may OOM."
)
return
else:
stable_count = 0
last_used = used
time.sleep(1.0)
# Timeout - log clearly so CI failures are diagnosable
final_used = self._get_gpu_memory_used()
final_gb = final_used / 1e9 if final_used else 0.0
raise RuntimeError(
f"[RemoteOpenAIServer] GPU memory did not stabilize within {timeout}s. "
f"Last reading: {last_reading:.2f} GB. "
"Child processes may still be holding GPU memory."
f"[RemoteOpenAIServer] GPU memory did not release within "
f"{timeout}s. Current: {final_gb:.2f} GB, "
f"target: {target / 1e9:.2f} GB, "
f"baseline: {baseline / 1e9:.2f} GB. "
f"Child processes may still be holding GPU memory."
)
def _poll(self) -> int | None: