From 3f5a1e1733200760169ff31ebe60a271072b199e Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Wed, 24 Jun 2026 23:18:57 -0500 Subject: [PATCH] [ROCm][CI] Expand basic correctness target suites (#46573) Signed-off-by: Andreas Karatzas Signed-off-by: Matthew Wong Co-authored-by: Matthew Wong Co-authored-by: Matt <156021403+mawong-amd@users.noreply.github.com> --- .buildkite/test-amd.yaml | 8 +- .../test_basic_correctness.py | 83 ++++++++++-- tests/conftest.py | 25 +++- tests/utils.py | 118 +++++++++++------- 4 files changed, 174 insertions(+), 60 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 7e48a12507..622ce38619 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -132,7 +132,7 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py commands: - - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' + - TARGET_TEST_SUITE=MI250 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' - pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)' - pytest models/language -v -s -m 'distributed(num_gpus=2)' @@ -646,7 +646,7 @@ steps: commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s basic_correctness/test_mem.py - - pytest -v -s basic_correctness/test_basic_correctness.py + - VLLM_TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py - pytest -v -s basic_correctness/test_cpu_offload.py - label: Distributed Model Tests (2 GPUs) # TBD @@ -668,7 +668,7 @@ steps: - tests/model_executor/model_loader/test_sharded_state_loader.py - tests/models/ commands: - - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' + - TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' - pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)' - pytest models/language -v -s -m 'distributed(num_gpus=2)' @@ -887,7 +887,7 @@ steps: commands: - pytest -v -s distributed/test_custom_all_reduce.py - torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py - - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' + - TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' - pytest -v -s -x lora/test_mixtral.py - label: Distributed Torchrun + Examples (4 GPUs) # TBD diff --git a/tests/basic_correctness/test_basic_correctness.py b/tests/basic_correctness/test_basic_correctness.py index 1a07ac6da6..810a3a0aee 100644 --- a/tests/basic_correctness/test_basic_correctness.py +++ b/tests/basic_correctness/test_basic_correctness.py @@ -29,7 +29,53 @@ MODELS = [ "meta-llama/Llama-3.2-1B-Instruct", ] -TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4") +TARGET_TEST_SUITE_ENV = "VLLM_TARGET_TEST_SUITE" +LEGACY_TARGET_TEST_SUITE_ENV = "TARGET_TEST_SUITE" + +GENERIC_DISTRIBUTED_TEST_SUITES = ("L4", "MI250", "MI300", "MI325", "MI355") +ALL_DISTRIBUTED_TEST_SUITES = (*GENERIC_DISTRIBUTED_TEST_SUITES, "A100") + + +def _default_target_test_suite() -> str: + if not current_platform.is_rocm(): + return "L4" + + try: + device_name = current_platform.get_device_name().upper() + except Exception: + device_name = "" + + if "MI355" in device_name: + return "MI355" + if "MI300" in device_name: + return "MI300" + if "MI325" in device_name: + return "MI325" + if "MI250" in device_name: + return "MI250" + + try: + from vllm.platforms import rocm as rocm_platform + + if rocm_platform.on_gfx950(): + return "MI355" + if rocm_platform.on_gfx942(): + return "MI300" + except Exception: + pass + + return "MI250" + + +def _resolve_target_test_suite() -> str: + for env_name in (TARGET_TEST_SUITE_ENV, LEGACY_TARGET_TEST_SUITE_ENV): + value = os.environ.get(env_name, "").strip().upper() + if value: + return value + return _default_target_test_suite() + + +TARGET_TEST_SUITE = _resolve_target_test_suite() def test_vllm_gc_ed(): @@ -131,14 +177,27 @@ def test_models( @multi_gpu_test(num_gpus=2) @pytest.mark.parametrize( - "model, distributed_executor_backend, attention_backend, test_suite, extra_env", + ( + "model, distributed_executor_backend, attention_backend, " + "target_test_suites, extra_env" + ), [ - ("facebook/opt-125m", "ray", "", "L4", {}), - ("facebook/opt-125m", "mp", "", "L4", {}), - ("meta-llama/Llama-3.2-1B-Instruct", "ray", "", "L4", {}), - ("meta-llama/Llama-3.2-1B-Instruct", "mp", "", "L4", {}), - ("facebook/opt-125m", "ray", "", "A100", {}), - ("facebook/opt-125m", "mp", "", "A100", {}), + ("facebook/opt-125m", "ray", "", ALL_DISTRIBUTED_TEST_SUITES, {}), + ("facebook/opt-125m", "mp", "", ALL_DISTRIBUTED_TEST_SUITES, {}), + ( + "meta-llama/Llama-3.2-1B-Instruct", + "ray", + "", + GENERIC_DISTRIBUTED_TEST_SUITES, + {}, + ), + ( + "meta-llama/Llama-3.2-1B-Instruct", + "mp", + "", + GENERIC_DISTRIBUTED_TEST_SUITES, + {}, + ), ], ) @pytest.mark.parametrize("enable_prompt_embeds", [True, False]) @@ -150,19 +209,19 @@ def test_models_distributed( model: str, distributed_executor_backend: str, attention_backend: str, - test_suite: str, + target_test_suites: tuple[str, ...], extra_env: dict[str, str], enable_prompt_embeds: bool, ) -> None: - if test_suite != TARGET_TEST_SUITE: - pytest.skip(f"Skip test for {test_suite}") + if TARGET_TEST_SUITE and TARGET_TEST_SUITE not in target_test_suites: + pytest.skip(f"Skip test for {TARGET_TEST_SUITE}") with monkeypatch.context() as monkeypatch_context: if ( model == "meta-llama/Llama-3.2-1B-Instruct" and distributed_executor_backend == "ray" and attention_backend == "" - and test_suite == "L4" + and TARGET_TEST_SUITE == "L4" and enable_prompt_embeds ): # noqa pytest.skip("enable_prompt_embeds does not work with ray compiled dag.") diff --git a/tests/conftest.py b/tests/conftest.py index 4fc43ef04b..4b92f285fa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -64,6 +64,7 @@ from vllm.logprobs import Logprob from vllm.multimodal.media import MediaWithBytes from vllm.multimodal.utils import fetch_image from vllm.outputs import RequestOutput +from vllm.platforms import current_platform from vllm.sampling_params import BeamSearchParams from vllm.transformers_utils.utils import maybe_model_redirect from vllm.utils.collection_utils import is_list_of @@ -852,6 +853,24 @@ class HfRunner: return self.model.predict(prompts, *args, convert_to_tensor=True, **kwargs) def __enter__(self): + if current_platform.is_rocm(): + # Record starting memory usage stats on ROCm so that we can wait for + # memory to roughly settle back below these levels on shutdown. This is + # helpful in cases where the HfRunner is initialized after significant GPU + # memory is already occupied, e.g. in + # tests/basic_correctness/test_basic_correctness.py::test_models_distributed + from tests.utils import ( + get_physical_device_indices, + record_gpu_memory_usage_stats, + ) + + if (device_count := current_platform.device_count()) > 0: + devices = get_physical_device_indices(devices=list(range(device_count))) + mem_usage_stats = record_gpu_memory_usage_stats(devices=devices) + self.threshold_ratios = { + device: 0.05 + mem_used / mem_tot + for device, (mem_used, mem_tot) in mem_usage_stats.items() + } return self def __exit__(self, exc_type, exc_value, traceback): @@ -861,7 +880,11 @@ class HfRunner: cleanup_dist_env_and_memory() # ROCm frees VRAM lazily; wait so a runner started right after this HF # model exits does not OOM on its startup memory guard. - wait_for_rocm_memory_to_settle() + wait_for_rocm_memory_to_settle( + threshold_ratio=getattr(self, "threshold_ratios", None) + ) + if hasattr(self, "threshold_ratios"): + del self.threshold_ratios @pytest.fixture(scope="session") diff --git a/tests/utils.py b/tests/utils.py index db5905b927..2acb971630 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1464,7 +1464,7 @@ def error_on_warning(category: type[Warning] = Warning): yield -def get_physical_device_indices(devices): +def get_physical_device_indices(devices: list[int]): visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES") if visible_devices is None: return devices @@ -1475,82 +1475,114 @@ def get_physical_device_indices(devices): @_nvml() +def record_gpu_memory_usage_stats( + *, + devices: list[int], +) -> dict[int, tuple[float, float]]: + output: dict[int, tuple[float, float]] = {} + for device in devices: + if current_platform.is_rocm(): + dev_handle = amdsmi_get_processor_handles()[device] + mem_info = amdsmi_get_gpu_vram_usage(dev_handle) + gb_used = mem_info["vram_used"] / 2**10 + gb_total = mem_info["vram_total"] / 2**10 + else: + dev_handle = nvmlDeviceGetHandleByIndex(device) + mem_info = nvmlDeviceGetMemoryInfo(dev_handle) + gb_used = mem_info.used / 2**30 + gb_total = mem_info.total / 2**30 + output[device] = (gb_used, gb_total) + return output + + def wait_for_gpu_memory_to_clear( *, devices: list[int], - threshold_bytes: int | None = None, - threshold_ratio: float | None = None, + threshold_bytes: int | dict[int, int] | None = None, + threshold_ratio: float | dict[int, float] | None = None, timeout_s: float = 120, ) -> None: assert threshold_bytes is not None or threshold_ratio is not None - if ( - current_platform.is_rocm() - and threshold_ratio is not None - and threshold_ratio < 0.05 - ): + devices = get_physical_device_indices(devices) + if isinstance(threshold_bytes, int): + threshold_bytes = {device: threshold_bytes for device in devices} + elif isinstance(threshold_bytes, dict): + assert threshold_bytes.keys() == set(devices) + if isinstance(threshold_ratio, float): + threshold_ratio = {device: threshold_ratio for device in devices} + elif isinstance(threshold_ratio, dict): + assert threshold_ratio.keys() == set(devices) + if current_platform.is_rocm() and threshold_ratio is not None: # ROCm can keep a small runtime/driver footprint resident even after # all model allocations are gone. On MI300 this has been observed # around 2.5 GiB, which is above a strict 1% idle threshold but nowhere # near the amount of free memory needed by the next vLLM runner. - min_threshold_bytes = 4 * 1024**3 - threshold_bytes = max(threshold_bytes or 0, min_threshold_bytes) + min_threshold_b = 4 * 1024**3 + if threshold_bytes is None: + threshold_bytes = {} + for device, ratio in threshold_ratio.items(): + threshold_bytes[device] = max( + threshold_bytes.get(device, 0), min_threshold_b if ratio < 0.05 else 0 + ) # Use nvml instead of pytorch to reduce measurement error from torch cuda # context. - devices = get_physical_device_indices(devices) start_time = time.time() while True: - output: dict[int, str] = {} - output_raw: dict[int, tuple[float, float]] = {} - for device in devices: - if current_platform.is_rocm(): - dev_handle = amdsmi_get_processor_handles()[device] - mem_info = amdsmi_get_gpu_vram_usage(dev_handle) - gb_used = mem_info["vram_used"] / 2**10 - gb_total = mem_info["vram_total"] / 2**10 - else: - dev_handle = nvmlDeviceGetHandleByIndex(device) - mem_info = nvmlDeviceGetMemoryInfo(dev_handle) - gb_used = mem_info.used / 2**30 - gb_total = mem_info.total / 2**30 - output_raw[device] = (gb_used, gb_total) - output[device] = f"{gb_used:.02f}/{gb_total:.02f}" - + output_raw = record_gpu_memory_usage_stats(devices=devices) + output = { + device: f"{gb_used:.02f}/{gb_total:.02f}" + for device, (gb_used, gb_total) in output_raw.items() + } print("gpu memory used/total (GiB): ", end="") for k, v in output.items(): print(f"{k}={v}; ", end="") print("") if threshold_bytes is not None and threshold_ratio is not None: - threshold_gib = threshold_bytes / 2**30 - threshold = f"max({threshold_gib:.2f} GiB, {threshold_ratio:.3f})" + threshold_gib = { + device: threshold_b / 2**30 + for device, threshold_b in threshold_bytes.items() + } + threshold = "; ".join( + f"{device=}: max({threshold_gib[device]:.2f} GiB, " + f"{threshold_ratio[device]:.3f})" + for device in devices + ) all_free = all( - used <= max(threshold_gib, total * threshold_ratio) - for used, total in output_raw.values() + used <= max(threshold_gib[device], total * threshold_ratio[device]) + for device, (used, total) in output_raw.items() ) elif threshold_bytes is not None: - threshold_gib = threshold_bytes / 2**30 - threshold = f"{threshold_gib} GiB" - all_free = all(used <= threshold_gib for used, _ in output_raw.values()) + threshold_gib = { + device: threshold_b / 2**30 + for device, threshold_b in threshold_bytes.items() + } + threshold = "; ".join( + f"{device=}: {threshold_gib[device]:.2f} GiB" for device in devices + ) + all_free = all( + used <= threshold_gib[device] + for device, (used, _) in output_raw.items() + ) else: assert threshold_ratio is not None - threshold = f"{threshold_ratio:.3f}" + threshold = "; ".join( + f"{device=}: {threshold_ratio[device]:.3f}" for device in devices + ) all_free = all( - used / total <= threshold_ratio for used, total in output_raw.values() + used / total <= threshold_ratio[device] + for device, (used, total) in output_raw.items() ) dur_s = time.time() - start_time if all_free: - print( - f"Done waiting for free GPU memory on devices {devices=} " - f"({threshold=}) {dur_s=:.02f}" - ) + print(f"Done waiting for free GPU memory on ({threshold=}) {dur_s=:.02f}") break if dur_s >= timeout_s: raise ValueError( - f"Memory of devices {devices=} not free after " - f"{dur_s=:.02f} ({threshold=})" + f"Memory of devices not free after {dur_s=:.02f} ({threshold=})" ) time.sleep(5) @@ -1558,7 +1590,7 @@ def wait_for_gpu_memory_to_clear( def wait_for_rocm_memory_to_settle( *, - threshold_ratio: float = 0.1, + threshold_ratio: float | dict[int, float] | None = 0.1, timeout_s: float = 240, ) -> None: """Block until ROCm device VRAM usage drops below ``threshold_ratio``.