[ROCm][CI] Expand basic correctness target suites (#46573)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Matt <156021403+mawong-amd@users.noreply.github.com>
This commit is contained in:
Andreas Karatzas
2026-06-25 12:18:57 +08:00
committed by GitHub
co-authored by Matthew Wong Matt
parent 710ebaa189
commit 3f5a1e1733
4 changed files with 174 additions and 60 deletions
+4 -4
View File
@@ -132,7 +132,7 @@ steps:
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- TARGET_TEST_SUITE=MI250 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
- pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
- pytest models/language -v -s -m 'distributed(num_gpus=2)'
@@ -646,7 +646,7 @@ steps:
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s basic_correctness/test_mem.py
- pytest -v -s basic_correctness/test_basic_correctness.py
- VLLM_TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py
- pytest -v -s basic_correctness/test_cpu_offload.py
- label: Distributed Model Tests (2 GPUs) # TBD
@@ -668,7 +668,7 @@ steps:
- tests/model_executor/model_loader/test_sharded_state_loader.py
- tests/models/
commands:
- TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
- pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
- pytest models/language -v -s -m 'distributed(num_gpus=2)'
@@ -887,7 +887,7 @@ steps:
commands:
- pytest -v -s distributed/test_custom_all_reduce.py
- torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py
- TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- pytest -v -s -x lora/test_mixtral.py
- label: Distributed Torchrun + Examples (4 GPUs) # TBD
@@ -29,7 +29,53 @@ MODELS = [
"meta-llama/Llama-3.2-1B-Instruct",
]
TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
TARGET_TEST_SUITE_ENV = "VLLM_TARGET_TEST_SUITE"
LEGACY_TARGET_TEST_SUITE_ENV = "TARGET_TEST_SUITE"
GENERIC_DISTRIBUTED_TEST_SUITES = ("L4", "MI250", "MI300", "MI325", "MI355")
ALL_DISTRIBUTED_TEST_SUITES = (*GENERIC_DISTRIBUTED_TEST_SUITES, "A100")
def _default_target_test_suite() -> str:
if not current_platform.is_rocm():
return "L4"
try:
device_name = current_platform.get_device_name().upper()
except Exception:
device_name = ""
if "MI355" in device_name:
return "MI355"
if "MI300" in device_name:
return "MI300"
if "MI325" in device_name:
return "MI325"
if "MI250" in device_name:
return "MI250"
try:
from vllm.platforms import rocm as rocm_platform
if rocm_platform.on_gfx950():
return "MI355"
if rocm_platform.on_gfx942():
return "MI300"
except Exception:
pass
return "MI250"
def _resolve_target_test_suite() -> str:
for env_name in (TARGET_TEST_SUITE_ENV, LEGACY_TARGET_TEST_SUITE_ENV):
value = os.environ.get(env_name, "").strip().upper()
if value:
return value
return _default_target_test_suite()
TARGET_TEST_SUITE = _resolve_target_test_suite()
def test_vllm_gc_ed():
@@ -131,14 +177,27 @@ def test_models(
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model, distributed_executor_backend, attention_backend, test_suite, extra_env",
(
"model, distributed_executor_backend, attention_backend, "
"target_test_suites, extra_env"
),
[
("facebook/opt-125m", "ray", "", "L4", {}),
("facebook/opt-125m", "mp", "", "L4", {}),
("meta-llama/Llama-3.2-1B-Instruct", "ray", "", "L4", {}),
("meta-llama/Llama-3.2-1B-Instruct", "mp", "", "L4", {}),
("facebook/opt-125m", "ray", "", "A100", {}),
("facebook/opt-125m", "mp", "", "A100", {}),
("facebook/opt-125m", "ray", "", ALL_DISTRIBUTED_TEST_SUITES, {}),
("facebook/opt-125m", "mp", "", ALL_DISTRIBUTED_TEST_SUITES, {}),
(
"meta-llama/Llama-3.2-1B-Instruct",
"ray",
"",
GENERIC_DISTRIBUTED_TEST_SUITES,
{},
),
(
"meta-llama/Llama-3.2-1B-Instruct",
"mp",
"",
GENERIC_DISTRIBUTED_TEST_SUITES,
{},
),
],
)
@pytest.mark.parametrize("enable_prompt_embeds", [True, False])
@@ -150,19 +209,19 @@ def test_models_distributed(
model: str,
distributed_executor_backend: str,
attention_backend: str,
test_suite: str,
target_test_suites: tuple[str, ...],
extra_env: dict[str, str],
enable_prompt_embeds: bool,
) -> None:
if test_suite != TARGET_TEST_SUITE:
pytest.skip(f"Skip test for {test_suite}")
if TARGET_TEST_SUITE and TARGET_TEST_SUITE not in target_test_suites:
pytest.skip(f"Skip test for {TARGET_TEST_SUITE}")
with monkeypatch.context() as monkeypatch_context:
if (
model == "meta-llama/Llama-3.2-1B-Instruct"
and distributed_executor_backend == "ray"
and attention_backend == ""
and test_suite == "L4"
and TARGET_TEST_SUITE == "L4"
and enable_prompt_embeds
): # noqa
pytest.skip("enable_prompt_embeds does not work with ray compiled dag.")
+24 -1
View File
@@ -64,6 +64,7 @@ from vllm.logprobs import Logprob
from vllm.multimodal.media import MediaWithBytes
from vllm.multimodal.utils import fetch_image
from vllm.outputs import RequestOutput
from vllm.platforms import current_platform
from vllm.sampling_params import BeamSearchParams
from vllm.transformers_utils.utils import maybe_model_redirect
from vllm.utils.collection_utils import is_list_of
@@ -852,6 +853,24 @@ class HfRunner:
return self.model.predict(prompts, *args, convert_to_tensor=True, **kwargs)
def __enter__(self):
if current_platform.is_rocm():
# Record starting memory usage stats on ROCm so that we can wait for
# memory to roughly settle back below these levels on shutdown. This is
# helpful in cases where the HfRunner is initialized after significant GPU
# memory is already occupied, e.g. in
# tests/basic_correctness/test_basic_correctness.py::test_models_distributed
from tests.utils import (
get_physical_device_indices,
record_gpu_memory_usage_stats,
)
if (device_count := current_platform.device_count()) > 0:
devices = get_physical_device_indices(devices=list(range(device_count)))
mem_usage_stats = record_gpu_memory_usage_stats(devices=devices)
self.threshold_ratios = {
device: 0.05 + mem_used / mem_tot
for device, (mem_used, mem_tot) in mem_usage_stats.items()
}
return self
def __exit__(self, exc_type, exc_value, traceback):
@@ -861,7 +880,11 @@ class HfRunner:
cleanup_dist_env_and_memory()
# ROCm frees VRAM lazily; wait so a runner started right after this HF
# model exits does not OOM on its startup memory guard.
wait_for_rocm_memory_to_settle()
wait_for_rocm_memory_to_settle(
threshold_ratio=getattr(self, "threshold_ratios", None)
)
if hasattr(self, "threshold_ratios"):
del self.threshold_ratios
@pytest.fixture(scope="session")
+75 -43
View File
@@ -1464,7 +1464,7 @@ def error_on_warning(category: type[Warning] = Warning):
yield
def get_physical_device_indices(devices):
def get_physical_device_indices(devices: list[int]):
visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES")
if visible_devices is None:
return devices
@@ -1475,82 +1475,114 @@ def get_physical_device_indices(devices):
@_nvml()
def record_gpu_memory_usage_stats(
*,
devices: list[int],
) -> dict[int, tuple[float, float]]:
output: dict[int, tuple[float, float]] = {}
for device in devices:
if current_platform.is_rocm():
dev_handle = amdsmi_get_processor_handles()[device]
mem_info = amdsmi_get_gpu_vram_usage(dev_handle)
gb_used = mem_info["vram_used"] / 2**10
gb_total = mem_info["vram_total"] / 2**10
else:
dev_handle = nvmlDeviceGetHandleByIndex(device)
mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
gb_used = mem_info.used / 2**30
gb_total = mem_info.total / 2**30
output[device] = (gb_used, gb_total)
return output
def wait_for_gpu_memory_to_clear(
*,
devices: list[int],
threshold_bytes: int | None = None,
threshold_ratio: float | None = None,
threshold_bytes: int | dict[int, int] | None = None,
threshold_ratio: float | dict[int, float] | None = None,
timeout_s: float = 120,
) -> None:
assert threshold_bytes is not None or threshold_ratio is not None
if (
current_platform.is_rocm()
and threshold_ratio is not None
and threshold_ratio < 0.05
):
devices = get_physical_device_indices(devices)
if isinstance(threshold_bytes, int):
threshold_bytes = {device: threshold_bytes for device in devices}
elif isinstance(threshold_bytes, dict):
assert threshold_bytes.keys() == set(devices)
if isinstance(threshold_ratio, float):
threshold_ratio = {device: threshold_ratio for device in devices}
elif isinstance(threshold_ratio, dict):
assert threshold_ratio.keys() == set(devices)
if current_platform.is_rocm() and threshold_ratio is not None:
# ROCm can keep a small runtime/driver footprint resident even after
# all model allocations are gone. On MI300 this has been observed
# around 2.5 GiB, which is above a strict 1% idle threshold but nowhere
# near the amount of free memory needed by the next vLLM runner.
min_threshold_bytes = 4 * 1024**3
threshold_bytes = max(threshold_bytes or 0, min_threshold_bytes)
min_threshold_b = 4 * 1024**3
if threshold_bytes is None:
threshold_bytes = {}
for device, ratio in threshold_ratio.items():
threshold_bytes[device] = max(
threshold_bytes.get(device, 0), min_threshold_b if ratio < 0.05 else 0
)
# Use nvml instead of pytorch to reduce measurement error from torch cuda
# context.
devices = get_physical_device_indices(devices)
start_time = time.time()
while True:
output: dict[int, str] = {}
output_raw: dict[int, tuple[float, float]] = {}
for device in devices:
if current_platform.is_rocm():
dev_handle = amdsmi_get_processor_handles()[device]
mem_info = amdsmi_get_gpu_vram_usage(dev_handle)
gb_used = mem_info["vram_used"] / 2**10
gb_total = mem_info["vram_total"] / 2**10
else:
dev_handle = nvmlDeviceGetHandleByIndex(device)
mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
gb_used = mem_info.used / 2**30
gb_total = mem_info.total / 2**30
output_raw[device] = (gb_used, gb_total)
output[device] = f"{gb_used:.02f}/{gb_total:.02f}"
output_raw = record_gpu_memory_usage_stats(devices=devices)
output = {
device: f"{gb_used:.02f}/{gb_total:.02f}"
for device, (gb_used, gb_total) in output_raw.items()
}
print("gpu memory used/total (GiB): ", end="")
for k, v in output.items():
print(f"{k}={v}; ", end="")
print("")
if threshold_bytes is not None and threshold_ratio is not None:
threshold_gib = threshold_bytes / 2**30
threshold = f"max({threshold_gib:.2f} GiB, {threshold_ratio:.3f})"
threshold_gib = {
device: threshold_b / 2**30
for device, threshold_b in threshold_bytes.items()
}
threshold = "; ".join(
f"{device=}: max({threshold_gib[device]:.2f} GiB, "
f"{threshold_ratio[device]:.3f})"
for device in devices
)
all_free = all(
used <= max(threshold_gib, total * threshold_ratio)
for used, total in output_raw.values()
used <= max(threshold_gib[device], total * threshold_ratio[device])
for device, (used, total) in output_raw.items()
)
elif threshold_bytes is not None:
threshold_gib = threshold_bytes / 2**30
threshold = f"{threshold_gib} GiB"
all_free = all(used <= threshold_gib for used, _ in output_raw.values())
threshold_gib = {
device: threshold_b / 2**30
for device, threshold_b in threshold_bytes.items()
}
threshold = "; ".join(
f"{device=}: {threshold_gib[device]:.2f} GiB" for device in devices
)
all_free = all(
used <= threshold_gib[device]
for device, (used, _) in output_raw.items()
)
else:
assert threshold_ratio is not None
threshold = f"{threshold_ratio:.3f}"
threshold = "; ".join(
f"{device=}: {threshold_ratio[device]:.3f}" for device in devices
)
all_free = all(
used / total <= threshold_ratio for used, total in output_raw.values()
used / total <= threshold_ratio[device]
for device, (used, total) in output_raw.items()
)
dur_s = time.time() - start_time
if all_free:
print(
f"Done waiting for free GPU memory on devices {devices=} "
f"({threshold=}) {dur_s=:.02f}"
)
print(f"Done waiting for free GPU memory on ({threshold=}) {dur_s=:.02f}")
break
if dur_s >= timeout_s:
raise ValueError(
f"Memory of devices {devices=} not free after "
f"{dur_s=:.02f} ({threshold=})"
f"Memory of devices not free after {dur_s=:.02f} ({threshold=})"
)
time.sleep(5)
@@ -1558,7 +1590,7 @@ def wait_for_gpu_memory_to_clear(
def wait_for_rocm_memory_to_settle(
*,
threshold_ratio: float = 0.1,
threshold_ratio: float | dict[int, float] | None = 0.1,
timeout_s: float = 240,
) -> None:
"""Block until ROCm device VRAM usage drops below ``threshold_ratio``.