[ROCm][CI] Expand basic correctness target suites (#46573)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Matt <156021403+mawong-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Matthew Wong
Matt
parent
710ebaa189
commit
3f5a1e1733
+24
-1
@@ -64,6 +64,7 @@ from vllm.logprobs import Logprob
|
||||
from vllm.multimodal.media import MediaWithBytes
|
||||
from vllm.multimodal.utils import fetch_image
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import BeamSearchParams
|
||||
from vllm.transformers_utils.utils import maybe_model_redirect
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
@@ -852,6 +853,24 @@ class HfRunner:
|
||||
return self.model.predict(prompts, *args, convert_to_tensor=True, **kwargs)
|
||||
|
||||
def __enter__(self):
|
||||
if current_platform.is_rocm():
|
||||
# Record starting memory usage stats on ROCm so that we can wait for
|
||||
# memory to roughly settle back below these levels on shutdown. This is
|
||||
# helpful in cases where the HfRunner is initialized after significant GPU
|
||||
# memory is already occupied, e.g. in
|
||||
# tests/basic_correctness/test_basic_correctness.py::test_models_distributed
|
||||
from tests.utils import (
|
||||
get_physical_device_indices,
|
||||
record_gpu_memory_usage_stats,
|
||||
)
|
||||
|
||||
if (device_count := current_platform.device_count()) > 0:
|
||||
devices = get_physical_device_indices(devices=list(range(device_count)))
|
||||
mem_usage_stats = record_gpu_memory_usage_stats(devices=devices)
|
||||
self.threshold_ratios = {
|
||||
device: 0.05 + mem_used / mem_tot
|
||||
for device, (mem_used, mem_tot) in mem_usage_stats.items()
|
||||
}
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
@@ -861,7 +880,11 @@ class HfRunner:
|
||||
cleanup_dist_env_and_memory()
|
||||
# ROCm frees VRAM lazily; wait so a runner started right after this HF
|
||||
# model exits does not OOM on its startup memory guard.
|
||||
wait_for_rocm_memory_to_settle()
|
||||
wait_for_rocm_memory_to_settle(
|
||||
threshold_ratio=getattr(self, "threshold_ratios", None)
|
||||
)
|
||||
if hasattr(self, "threshold_ratios"):
|
||||
del self.threshold_ratios
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
|
||||
Reference in New Issue
Block a user