From b74233860d4f2419ac5a763f6c83859bfe742ad0 Mon Sep 17 00:00:00 2001 From: stefankoncarevic Date: Fri, 9 Oct 2026 18:48:43 +0200 Subject: [PATCH] [CI][ROCm] Wait for VRAM to settle before building LLM in test_full_graph (#60872) Signed-off-by: Stefan Koncarevic --- tests/compile/fullgraph/test_full_graph.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/compile/fullgraph/test_full_graph.py b/tests/compile/fullgraph/test_full_graph.py index 4ce7d6767f..fceed6346d 100644 --- a/tests/compile/fullgraph/test_full_graph.py +++ b/tests/compile/fullgraph/test_full_graph.py @@ -10,11 +10,17 @@ import torch from tests.quantization.utils import is_quant_method_supported from vllm import LLM, SamplingParams -from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode, PassConfig +from vllm.config import ( + CacheConfig, + CompilationConfig, + CompilationMode, + CUDAGraphMode, + PassConfig, +) from vllm.platforms import current_platform from vllm.utils.torch_utils import is_torch_equal_or_newer -from ...utils import create_new_process_for_each_test +from ...utils import create_new_process_for_each_test, wait_for_memory_to_settle def models_list(*, all: bool = True, keywords: list[str] | None = None): @@ -173,6 +179,14 @@ def run_model( if compilation_config.cudagraph_mode is None: compilation_config.cudagraph_mode = CUDAGraphMode.NONE + # V1 startup requires free_memory >= total * gpu_memory_utilization, so a + # device that is not free yet fails engine init rather than waiting for it. + # VllmRunner guards this; building LLM directly does not. No-op off ROCm. + wait_for_memory_to_settle( + threshold_ratio=1.0 + - model_kwargs.get("gpu_memory_utilization", CacheConfig.gpu_memory_utilization) + ) + llm = LLM( model=model, compilation_config=compilation_config,