[ROCm][CI] Patch conftest to resolve occasional OOMs (#45722)
Signed-off-by: Micah Williamson <micah.williamson@amd.com>
This commit is contained in:
@@ -51,6 +51,7 @@ from vllm import LLM, SamplingParams, envs
|
|||||||
from vllm.assets.audio import AudioAsset
|
from vllm.assets.audio import AudioAsset
|
||||||
from vllm.assets.image import ImageAsset
|
from vllm.assets.image import ImageAsset
|
||||||
from vllm.assets.video import VideoAsset
|
from vllm.assets.video import VideoAsset
|
||||||
|
from vllm.config.cache import CacheConfig
|
||||||
from vllm.config.model import ConvertOption, RunnerOption, _get_and_verify_dtype
|
from vllm.config.model import ConvertOption, RunnerOption, _get_and_verify_dtype
|
||||||
from vllm.connections import global_http_connection
|
from vllm.connections import global_http_connection
|
||||||
from vllm.distributed import (
|
from vllm.distributed import (
|
||||||
@@ -924,6 +925,20 @@ class VllmRunner:
|
|||||||
num_speculative_tokens + 1
|
num_speculative_tokens + 1
|
||||||
)
|
)
|
||||||
|
|
||||||
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
|
if current_platform.is_rocm():
|
||||||
|
gpu_memory_utilization = kwargs.get(
|
||||||
|
"gpu_memory_utilization",
|
||||||
|
CacheConfig.gpu_memory_utilization,
|
||||||
|
)
|
||||||
|
# V1 startup requires free_memory >= total * gpu_memory_utilization.
|
||||||
|
# ROCm CI can hand a test a device that is still lazily releasing
|
||||||
|
# VRAM from a previous process, so wait before constructing LLM.
|
||||||
|
from tests.utils import wait_for_rocm_memory_to_settle
|
||||||
|
|
||||||
|
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||||
|
|
||||||
with init_ctx:
|
with init_ctx:
|
||||||
self.llm = LLM(
|
self.llm = LLM(
|
||||||
model=model_name,
|
model=model_name,
|
||||||
|
|||||||
Reference in New Issue
Block a user