Stabilize GPU memory teardown between ROCm CI tests (#49242)

Signed-off-by: aarushjain29 <Aarushi.Jain2@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Aarushi Jain
2026-07-25 04:21:36 +00:00
committed by GitHub
co-authored by Andreas Karatzas Andreas Karatzas OpenAI Codex
parent aaaeda98dc
commit 0ba2aa35a8
3 changed files with 126 additions and 93 deletions
+26 -9
View File
@@ -1728,33 +1728,50 @@ def disable_deepgemm_ue8m0(monkeypatch):
is_deep_gemm_e8m0_used.cache_clear()
def _should_clean_gpu_memory_between_tests() -> bool:
setting = os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY")
if setting == "1":
return True
if setting == "0":
return False
# ROCm reclaims VRAM lazily; default to waiting between tests on ROCm CI.
return current_platform.is_rocm()
@pytest.fixture(autouse=True)
def clean_gpu_memory_between_tests():
if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1":
if not _should_clean_gpu_memory_between_tests():
yield
return
# Wait for GPU memory to be cleared before starting the test
import gc
from tests.utils import wait_for_gpu_memory_to_clear
from tests.utils import wait_for_gpu_memory_to_clear, wait_for_rocm_memory_to_settle
num_gpus = torch.accelerator.device_count()
if num_gpus > 0:
def _wait_for_settled_gpu_memory() -> None:
if num_gpus <= 0:
return
try:
wait_for_gpu_memory_to_clear(
devices=list(range(num_gpus)),
threshold_ratio=0.1,
)
if current_platform.is_rocm():
wait_for_rocm_memory_to_settle()
else:
wait_for_gpu_memory_to_clear(
devices=list(range(num_gpus)),
threshold_ratio=0.1,
)
except ValueError as e:
logger.info("Failed to clean GPU memory: %s", e)
_wait_for_settled_gpu_memory()
yield
# Clean up GPU memory after the test
if torch.cuda.is_available():
torch.accelerator.empty_cache()
gc.collect()
_wait_for_settled_gpu_memory()
@pytest.fixture