[ROCm][CI] Expand basic correctness target suites (#46573)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Matt <156021403+mawong-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Matthew Wong
Matt
parent
710ebaa189
commit
3f5a1e1733
@@ -132,7 +132,7 @@ steps:
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
|
||||
- TARGET_TEST_SUITE=MI250 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
|
||||
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
|
||||
- pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
|
||||
- pytest models/language -v -s -m 'distributed(num_gpus=2)'
|
||||
@@ -646,7 +646,7 @@ steps:
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s basic_correctness/test_mem.py
|
||||
- pytest -v -s basic_correctness/test_basic_correctness.py
|
||||
- VLLM_TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py
|
||||
- pytest -v -s basic_correctness/test_cpu_offload.py
|
||||
|
||||
- label: Distributed Model Tests (2 GPUs) # TBD
|
||||
@@ -668,7 +668,7 @@ steps:
|
||||
- tests/model_executor/model_loader/test_sharded_state_loader.py
|
||||
- tests/models/
|
||||
commands:
|
||||
- TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
|
||||
- TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
|
||||
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
|
||||
- pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
|
||||
- pytest models/language -v -s -m 'distributed(num_gpus=2)'
|
||||
@@ -887,7 +887,7 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s distributed/test_custom_all_reduce.py
|
||||
- torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py
|
||||
- TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
|
||||
- TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
|
||||
- pytest -v -s -x lora/test_mixtral.py
|
||||
|
||||
- label: Distributed Torchrun + Examples (4 GPUs) # TBD
|
||||
|
||||
@@ -29,7 +29,53 @@ MODELS = [
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
]
|
||||
|
||||
TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
|
||||
TARGET_TEST_SUITE_ENV = "VLLM_TARGET_TEST_SUITE"
|
||||
LEGACY_TARGET_TEST_SUITE_ENV = "TARGET_TEST_SUITE"
|
||||
|
||||
GENERIC_DISTRIBUTED_TEST_SUITES = ("L4", "MI250", "MI300", "MI325", "MI355")
|
||||
ALL_DISTRIBUTED_TEST_SUITES = (*GENERIC_DISTRIBUTED_TEST_SUITES, "A100")
|
||||
|
||||
|
||||
def _default_target_test_suite() -> str:
|
||||
if not current_platform.is_rocm():
|
||||
return "L4"
|
||||
|
||||
try:
|
||||
device_name = current_platform.get_device_name().upper()
|
||||
except Exception:
|
||||
device_name = ""
|
||||
|
||||
if "MI355" in device_name:
|
||||
return "MI355"
|
||||
if "MI300" in device_name:
|
||||
return "MI300"
|
||||
if "MI325" in device_name:
|
||||
return "MI325"
|
||||
if "MI250" in device_name:
|
||||
return "MI250"
|
||||
|
||||
try:
|
||||
from vllm.platforms import rocm as rocm_platform
|
||||
|
||||
if rocm_platform.on_gfx950():
|
||||
return "MI355"
|
||||
if rocm_platform.on_gfx942():
|
||||
return "MI300"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return "MI250"
|
||||
|
||||
|
||||
def _resolve_target_test_suite() -> str:
|
||||
for env_name in (TARGET_TEST_SUITE_ENV, LEGACY_TARGET_TEST_SUITE_ENV):
|
||||
value = os.environ.get(env_name, "").strip().upper()
|
||||
if value:
|
||||
return value
|
||||
return _default_target_test_suite()
|
||||
|
||||
|
||||
TARGET_TEST_SUITE = _resolve_target_test_suite()
|
||||
|
||||
|
||||
def test_vllm_gc_ed():
|
||||
@@ -131,14 +177,27 @@ def test_models(
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"model, distributed_executor_backend, attention_backend, test_suite, extra_env",
|
||||
(
|
||||
"model, distributed_executor_backend, attention_backend, "
|
||||
"target_test_suites, extra_env"
|
||||
),
|
||||
[
|
||||
("facebook/opt-125m", "ray", "", "L4", {}),
|
||||
("facebook/opt-125m", "mp", "", "L4", {}),
|
||||
("meta-llama/Llama-3.2-1B-Instruct", "ray", "", "L4", {}),
|
||||
("meta-llama/Llama-3.2-1B-Instruct", "mp", "", "L4", {}),
|
||||
("facebook/opt-125m", "ray", "", "A100", {}),
|
||||
("facebook/opt-125m", "mp", "", "A100", {}),
|
||||
("facebook/opt-125m", "ray", "", ALL_DISTRIBUTED_TEST_SUITES, {}),
|
||||
("facebook/opt-125m", "mp", "", ALL_DISTRIBUTED_TEST_SUITES, {}),
|
||||
(
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"ray",
|
||||
"",
|
||||
GENERIC_DISTRIBUTED_TEST_SUITES,
|
||||
{},
|
||||
),
|
||||
(
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"mp",
|
||||
"",
|
||||
GENERIC_DISTRIBUTED_TEST_SUITES,
|
||||
{},
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("enable_prompt_embeds", [True, False])
|
||||
@@ -150,19 +209,19 @@ def test_models_distributed(
|
||||
model: str,
|
||||
distributed_executor_backend: str,
|
||||
attention_backend: str,
|
||||
test_suite: str,
|
||||
target_test_suites: tuple[str, ...],
|
||||
extra_env: dict[str, str],
|
||||
enable_prompt_embeds: bool,
|
||||
) -> None:
|
||||
if test_suite != TARGET_TEST_SUITE:
|
||||
pytest.skip(f"Skip test for {test_suite}")
|
||||
if TARGET_TEST_SUITE and TARGET_TEST_SUITE not in target_test_suites:
|
||||
pytest.skip(f"Skip test for {TARGET_TEST_SUITE}")
|
||||
|
||||
with monkeypatch.context() as monkeypatch_context:
|
||||
if (
|
||||
model == "meta-llama/Llama-3.2-1B-Instruct"
|
||||
and distributed_executor_backend == "ray"
|
||||
and attention_backend == ""
|
||||
and test_suite == "L4"
|
||||
and TARGET_TEST_SUITE == "L4"
|
||||
and enable_prompt_embeds
|
||||
): # noqa
|
||||
pytest.skip("enable_prompt_embeds does not work with ray compiled dag.")
|
||||
|
||||
+24
-1
@@ -64,6 +64,7 @@ from vllm.logprobs import Logprob
|
||||
from vllm.multimodal.media import MediaWithBytes
|
||||
from vllm.multimodal.utils import fetch_image
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import BeamSearchParams
|
||||
from vllm.transformers_utils.utils import maybe_model_redirect
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
@@ -852,6 +853,24 @@ class HfRunner:
|
||||
return self.model.predict(prompts, *args, convert_to_tensor=True, **kwargs)
|
||||
|
||||
def __enter__(self):
|
||||
if current_platform.is_rocm():
|
||||
# Record starting memory usage stats on ROCm so that we can wait for
|
||||
# memory to roughly settle back below these levels on shutdown. This is
|
||||
# helpful in cases where the HfRunner is initialized after significant GPU
|
||||
# memory is already occupied, e.g. in
|
||||
# tests/basic_correctness/test_basic_correctness.py::test_models_distributed
|
||||
from tests.utils import (
|
||||
get_physical_device_indices,
|
||||
record_gpu_memory_usage_stats,
|
||||
)
|
||||
|
||||
if (device_count := current_platform.device_count()) > 0:
|
||||
devices = get_physical_device_indices(devices=list(range(device_count)))
|
||||
mem_usage_stats = record_gpu_memory_usage_stats(devices=devices)
|
||||
self.threshold_ratios = {
|
||||
device: 0.05 + mem_used / mem_tot
|
||||
for device, (mem_used, mem_tot) in mem_usage_stats.items()
|
||||
}
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
@@ -861,7 +880,11 @@ class HfRunner:
|
||||
cleanup_dist_env_and_memory()
|
||||
# ROCm frees VRAM lazily; wait so a runner started right after this HF
|
||||
# model exits does not OOM on its startup memory guard.
|
||||
wait_for_rocm_memory_to_settle()
|
||||
wait_for_rocm_memory_to_settle(
|
||||
threshold_ratio=getattr(self, "threshold_ratios", None)
|
||||
)
|
||||
if hasattr(self, "threshold_ratios"):
|
||||
del self.threshold_ratios
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
|
||||
+75
-43
@@ -1464,7 +1464,7 @@ def error_on_warning(category: type[Warning] = Warning):
|
||||
yield
|
||||
|
||||
|
||||
def get_physical_device_indices(devices):
|
||||
def get_physical_device_indices(devices: list[int]):
|
||||
visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES")
|
||||
if visible_devices is None:
|
||||
return devices
|
||||
@@ -1475,82 +1475,114 @@ def get_physical_device_indices(devices):
|
||||
|
||||
|
||||
@_nvml()
|
||||
def record_gpu_memory_usage_stats(
|
||||
*,
|
||||
devices: list[int],
|
||||
) -> dict[int, tuple[float, float]]:
|
||||
output: dict[int, tuple[float, float]] = {}
|
||||
for device in devices:
|
||||
if current_platform.is_rocm():
|
||||
dev_handle = amdsmi_get_processor_handles()[device]
|
||||
mem_info = amdsmi_get_gpu_vram_usage(dev_handle)
|
||||
gb_used = mem_info["vram_used"] / 2**10
|
||||
gb_total = mem_info["vram_total"] / 2**10
|
||||
else:
|
||||
dev_handle = nvmlDeviceGetHandleByIndex(device)
|
||||
mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
|
||||
gb_used = mem_info.used / 2**30
|
||||
gb_total = mem_info.total / 2**30
|
||||
output[device] = (gb_used, gb_total)
|
||||
return output
|
||||
|
||||
|
||||
def wait_for_gpu_memory_to_clear(
|
||||
*,
|
||||
devices: list[int],
|
||||
threshold_bytes: int | None = None,
|
||||
threshold_ratio: float | None = None,
|
||||
threshold_bytes: int | dict[int, int] | None = None,
|
||||
threshold_ratio: float | dict[int, float] | None = None,
|
||||
timeout_s: float = 120,
|
||||
) -> None:
|
||||
assert threshold_bytes is not None or threshold_ratio is not None
|
||||
if (
|
||||
current_platform.is_rocm()
|
||||
and threshold_ratio is not None
|
||||
and threshold_ratio < 0.05
|
||||
):
|
||||
devices = get_physical_device_indices(devices)
|
||||
if isinstance(threshold_bytes, int):
|
||||
threshold_bytes = {device: threshold_bytes for device in devices}
|
||||
elif isinstance(threshold_bytes, dict):
|
||||
assert threshold_bytes.keys() == set(devices)
|
||||
if isinstance(threshold_ratio, float):
|
||||
threshold_ratio = {device: threshold_ratio for device in devices}
|
||||
elif isinstance(threshold_ratio, dict):
|
||||
assert threshold_ratio.keys() == set(devices)
|
||||
if current_platform.is_rocm() and threshold_ratio is not None:
|
||||
# ROCm can keep a small runtime/driver footprint resident even after
|
||||
# all model allocations are gone. On MI300 this has been observed
|
||||
# around 2.5 GiB, which is above a strict 1% idle threshold but nowhere
|
||||
# near the amount of free memory needed by the next vLLM runner.
|
||||
min_threshold_bytes = 4 * 1024**3
|
||||
threshold_bytes = max(threshold_bytes or 0, min_threshold_bytes)
|
||||
min_threshold_b = 4 * 1024**3
|
||||
if threshold_bytes is None:
|
||||
threshold_bytes = {}
|
||||
for device, ratio in threshold_ratio.items():
|
||||
threshold_bytes[device] = max(
|
||||
threshold_bytes.get(device, 0), min_threshold_b if ratio < 0.05 else 0
|
||||
)
|
||||
|
||||
# Use nvml instead of pytorch to reduce measurement error from torch cuda
|
||||
# context.
|
||||
devices = get_physical_device_indices(devices)
|
||||
start_time = time.time()
|
||||
while True:
|
||||
output: dict[int, str] = {}
|
||||
output_raw: dict[int, tuple[float, float]] = {}
|
||||
for device in devices:
|
||||
if current_platform.is_rocm():
|
||||
dev_handle = amdsmi_get_processor_handles()[device]
|
||||
mem_info = amdsmi_get_gpu_vram_usage(dev_handle)
|
||||
gb_used = mem_info["vram_used"] / 2**10
|
||||
gb_total = mem_info["vram_total"] / 2**10
|
||||
else:
|
||||
dev_handle = nvmlDeviceGetHandleByIndex(device)
|
||||
mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
|
||||
gb_used = mem_info.used / 2**30
|
||||
gb_total = mem_info.total / 2**30
|
||||
output_raw[device] = (gb_used, gb_total)
|
||||
output[device] = f"{gb_used:.02f}/{gb_total:.02f}"
|
||||
|
||||
output_raw = record_gpu_memory_usage_stats(devices=devices)
|
||||
output = {
|
||||
device: f"{gb_used:.02f}/{gb_total:.02f}"
|
||||
for device, (gb_used, gb_total) in output_raw.items()
|
||||
}
|
||||
print("gpu memory used/total (GiB): ", end="")
|
||||
for k, v in output.items():
|
||||
print(f"{k}={v}; ", end="")
|
||||
print("")
|
||||
|
||||
if threshold_bytes is not None and threshold_ratio is not None:
|
||||
threshold_gib = threshold_bytes / 2**30
|
||||
threshold = f"max({threshold_gib:.2f} GiB, {threshold_ratio:.3f})"
|
||||
threshold_gib = {
|
||||
device: threshold_b / 2**30
|
||||
for device, threshold_b in threshold_bytes.items()
|
||||
}
|
||||
threshold = "; ".join(
|
||||
f"{device=}: max({threshold_gib[device]:.2f} GiB, "
|
||||
f"{threshold_ratio[device]:.3f})"
|
||||
for device in devices
|
||||
)
|
||||
all_free = all(
|
||||
used <= max(threshold_gib, total * threshold_ratio)
|
||||
for used, total in output_raw.values()
|
||||
used <= max(threshold_gib[device], total * threshold_ratio[device])
|
||||
for device, (used, total) in output_raw.items()
|
||||
)
|
||||
elif threshold_bytes is not None:
|
||||
threshold_gib = threshold_bytes / 2**30
|
||||
threshold = f"{threshold_gib} GiB"
|
||||
all_free = all(used <= threshold_gib for used, _ in output_raw.values())
|
||||
threshold_gib = {
|
||||
device: threshold_b / 2**30
|
||||
for device, threshold_b in threshold_bytes.items()
|
||||
}
|
||||
threshold = "; ".join(
|
||||
f"{device=}: {threshold_gib[device]:.2f} GiB" for device in devices
|
||||
)
|
||||
all_free = all(
|
||||
used <= threshold_gib[device]
|
||||
for device, (used, _) in output_raw.items()
|
||||
)
|
||||
else:
|
||||
assert threshold_ratio is not None
|
||||
threshold = f"{threshold_ratio:.3f}"
|
||||
threshold = "; ".join(
|
||||
f"{device=}: {threshold_ratio[device]:.3f}" for device in devices
|
||||
)
|
||||
all_free = all(
|
||||
used / total <= threshold_ratio for used, total in output_raw.values()
|
||||
used / total <= threshold_ratio[device]
|
||||
for device, (used, total) in output_raw.items()
|
||||
)
|
||||
|
||||
dur_s = time.time() - start_time
|
||||
if all_free:
|
||||
print(
|
||||
f"Done waiting for free GPU memory on devices {devices=} "
|
||||
f"({threshold=}) {dur_s=:.02f}"
|
||||
)
|
||||
print(f"Done waiting for free GPU memory on ({threshold=}) {dur_s=:.02f}")
|
||||
break
|
||||
|
||||
if dur_s >= timeout_s:
|
||||
raise ValueError(
|
||||
f"Memory of devices {devices=} not free after "
|
||||
f"{dur_s=:.02f} ({threshold=})"
|
||||
f"Memory of devices not free after {dur_s=:.02f} ({threshold=})"
|
||||
)
|
||||
|
||||
time.sleep(5)
|
||||
@@ -1558,7 +1590,7 @@ def wait_for_gpu_memory_to_clear(
|
||||
|
||||
def wait_for_rocm_memory_to_settle(
|
||||
*,
|
||||
threshold_ratio: float = 0.1,
|
||||
threshold_ratio: float | dict[int, float] | None = 0.1,
|
||||
timeout_s: float = 240,
|
||||
) -> None:
|
||||
"""Block until ROCm device VRAM usage drops below ``threshold_ratio``.
|
||||
|
||||
Reference in New Issue
Block a user