Stabilize GPU memory teardown between ROCm CI tests (#49242)

Signed-off-by: aarushjain29 <Aarushi.Jain2@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Aarushi Jain
2026-07-25 04:21:36 +00:00
committed by GitHub
co-authored by Andreas Karatzas Andreas Karatzas OpenAI Codex
parent aaaeda98dc
commit 0ba2aa35a8
3 changed files with 126 additions and 93 deletions
@@ -98,7 +98,7 @@ GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "")
# Trap the SIGINT signal (triggered by Ctrl+C)
trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT
# Waits for vLLM to start.
wait_for_server() {
@@ -110,10 +110,20 @@ wait_for_server() {
}
# Function to clean up previous instances
wait_for_gpu_memory_release() {
if [[ "$SMI_BIN" == *"rocm"* ]]; then
PYTHONPATH="${GIT_ROOT}" python3 -c "from tests.utils import wait_for_rocm_memory_to_settle; wait_for_rocm_memory_to_settle()"
fi
}
cleanup_instances() {
echo "Cleaning up any running vLLM instances..."
pkill -f "vllm serve" || true
pkill -f "toy_proxy_server.py" || true
pkill -TERM -f "vllm serve" || true
sleep 3
pkill -9 -f "vllm serve" || true
sleep 2
wait_for_gpu_memory_release
}
get_num_gpus() {
@@ -132,6 +142,7 @@ get_num_gpus() {
# Function to run tests for a specific model
run_tests_for_model() {
local model_name=$1
cleanup_instances
echo "================================"
echo "Testing model: $model_name"
echo "================================"
@@ -303,7 +314,6 @@ run_tests_for_model() {
# Clean up before running next model
cleanup_instances
sleep 3
}
# Run tests for each model