Stabilize GPU memory teardown between ROCm CI tests (#49242)
Signed-off-by: aarushjain29 <Aarushi.Jain2@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Andreas Karatzas
Andreas Karatzas
OpenAI Codex
parent
aaaeda98dc
commit
0ba2aa35a8
+26
-9
@@ -1728,33 +1728,50 @@ def disable_deepgemm_ue8m0(monkeypatch):
|
|||||||
is_deep_gemm_e8m0_used.cache_clear()
|
is_deep_gemm_e8m0_used.cache_clear()
|
||||||
|
|
||||||
|
|
||||||
|
def _should_clean_gpu_memory_between_tests() -> bool:
|
||||||
|
setting = os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY")
|
||||||
|
if setting == "1":
|
||||||
|
return True
|
||||||
|
if setting == "0":
|
||||||
|
return False
|
||||||
|
# ROCm reclaims VRAM lazily; default to waiting between tests on ROCm CI.
|
||||||
|
return current_platform.is_rocm()
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
def clean_gpu_memory_between_tests():
|
def clean_gpu_memory_between_tests():
|
||||||
if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1":
|
if not _should_clean_gpu_memory_between_tests():
|
||||||
yield
|
yield
|
||||||
return
|
return
|
||||||
|
|
||||||
# Wait for GPU memory to be cleared before starting the test
|
|
||||||
import gc
|
import gc
|
||||||
|
|
||||||
from tests.utils import wait_for_gpu_memory_to_clear
|
from tests.utils import wait_for_gpu_memory_to_clear, wait_for_rocm_memory_to_settle
|
||||||
|
|
||||||
num_gpus = torch.accelerator.device_count()
|
num_gpus = torch.accelerator.device_count()
|
||||||
if num_gpus > 0:
|
|
||||||
|
def _wait_for_settled_gpu_memory() -> None:
|
||||||
|
if num_gpus <= 0:
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
wait_for_gpu_memory_to_clear(
|
if current_platform.is_rocm():
|
||||||
devices=list(range(num_gpus)),
|
wait_for_rocm_memory_to_settle()
|
||||||
threshold_ratio=0.1,
|
else:
|
||||||
)
|
wait_for_gpu_memory_to_clear(
|
||||||
|
devices=list(range(num_gpus)),
|
||||||
|
threshold_ratio=0.1,
|
||||||
|
)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
logger.info("Failed to clean GPU memory: %s", e)
|
logger.info("Failed to clean GPU memory: %s", e)
|
||||||
|
|
||||||
|
_wait_for_settled_gpu_memory()
|
||||||
|
|
||||||
yield
|
yield
|
||||||
|
|
||||||
# Clean up GPU memory after the test
|
|
||||||
if torch.cuda.is_available():
|
if torch.cuda.is_available():
|
||||||
torch.accelerator.empty_cache()
|
torch.accelerator.empty_cache()
|
||||||
gc.collect()
|
gc.collect()
|
||||||
|
_wait_for_settled_gpu_memory()
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
|||||||
+87
-81
@@ -7,6 +7,7 @@ from packaging.version import Version
|
|||||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
from transformers import __version__ as TRANSFORMERS_VERSION
|
||||||
|
|
||||||
import vllm
|
import vllm
|
||||||
|
from tests.conftest import VllmRunner
|
||||||
from vllm.assets.image import ImageAsset
|
from vllm.assets.image import ImageAsset
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
@@ -56,24 +57,29 @@ class Qwen2VLTester:
|
|||||||
|
|
||||||
def __init__(self, config: TestConfig):
|
def __init__(self, config: TestConfig):
|
||||||
self.config = config
|
self.config = config
|
||||||
self.llm = self._initialize_llm()
|
self._runner = VllmRunner(
|
||||||
|
model_name=config.model_path,
|
||||||
def _initialize_llm(self) -> vllm.LLM:
|
max_num_seqs=config.max_num_seqs,
|
||||||
"""Initialize the LLM with given configuration"""
|
|
||||||
return vllm.LLM(
|
|
||||||
model=self.config.model_path,
|
|
||||||
max_num_seqs=self.config.max_num_seqs,
|
|
||||||
enable_lora=True,
|
enable_lora=True,
|
||||||
max_loras=self.config.max_loras,
|
max_loras=config.max_loras,
|
||||||
max_lora_rank=self.config.max_lora_rank,
|
max_lora_rank=config.max_lora_rank,
|
||||||
enable_tower_connector_lora=self.config.enable_tower_connector_lora,
|
enable_tower_connector_lora=config.enable_tower_connector_lora,
|
||||||
trust_remote_code=True,
|
gpu_memory_utilization=config.gpu_memory_utilization,
|
||||||
gpu_memory_utilization=self.config.gpu_memory_utilization,
|
mm_processor_kwargs=config.mm_processor_kwargs,
|
||||||
mm_processor_kwargs=self.config.mm_processor_kwargs,
|
mm_processor_cache_gb=config.mm_processor_cache_gb,
|
||||||
mm_processor_cache_gb=self.config.mm_processor_cache_gb,
|
max_model_len=config.max_model_len,
|
||||||
max_model_len=self.config.max_model_len,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def llm(self) -> vllm.LLM:
|
||||||
|
return self._runner.get_llm()
|
||||||
|
|
||||||
|
def __enter__(self) -> "Qwen2VLTester":
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type, exc_value, traceback) -> None:
|
||||||
|
self._runner.__exit__(exc_type, exc_value, traceback)
|
||||||
|
|
||||||
def run_test(
|
def run_test(
|
||||||
self,
|
self,
|
||||||
images: list[ImageAsset],
|
images: list[ImageAsset],
|
||||||
@@ -183,29 +189,29 @@ QWEN3VL_MODEL_PATH = "Qwen/Qwen3-VL-4B-Instruct"
|
|||||||
def test_qwen2vl_lora(qwen2vl_lora_files):
|
def test_qwen2vl_lora(qwen2vl_lora_files):
|
||||||
"""Test Qwen 2.0 VL model with LoRA"""
|
"""Test Qwen 2.0 VL model with LoRA"""
|
||||||
config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files)
|
config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files)
|
||||||
tester = Qwen2VLTester(config)
|
with Qwen2VLTester(config) as tester:
|
||||||
|
# Test with different LoRA IDs
|
||||||
# Test with different LoRA IDs
|
for lora_id in [1, 2]:
|
||||||
for lora_id in [1, 2]:
|
tester.run_test(
|
||||||
tester.run_test(TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id)
|
TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_qwen2vl_lora_beam_search(qwen2vl_lora_files):
|
def test_qwen2vl_lora_beam_search(qwen2vl_lora_files):
|
||||||
"""Test Qwen 2.0 VL model with LoRA through beam search."""
|
"""Test Qwen 2.0 VL model with LoRA through beam search."""
|
||||||
config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files)
|
config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files)
|
||||||
tester = Qwen2VLTester(config)
|
with Qwen2VLTester(config) as tester:
|
||||||
|
# Test with different LoRA IDs
|
||||||
# Test with different LoRA IDs
|
for lora_id in [1, 2]:
|
||||||
for lora_id in [1, 2]:
|
# NOTE currently, we only test cherry blossom since stop sign
|
||||||
# NOTE currently, we only test cherry blossom since stop sign
|
# output is slightly different for v1; - the root cause is likely
|
||||||
# output is slightly different for v1; - the root cause is likely
|
# independent of the intent of this test, which is to ensure beam
|
||||||
# independent of the intent of this test, which is to ensure beam
|
# search passes through lora through correctly.
|
||||||
# search passes through lora through correctly.
|
tester.run_beam_search_test(
|
||||||
tester.run_beam_search_test(
|
[ImageAsset("cherry_blossom")],
|
||||||
[ImageAsset("cherry_blossom")],
|
expected_outputs=EXPECTED_BEAM_SEARCH_OUTPUTS,
|
||||||
expected_outputs=EXPECTED_BEAM_SEARCH_OUTPUTS,
|
lora_id=lora_id,
|
||||||
lora_id=lora_id,
|
)
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(
|
@pytest.mark.skipif(
|
||||||
@@ -214,11 +220,12 @@ def test_qwen2vl_lora_beam_search(qwen2vl_lora_files):
|
|||||||
def test_qwen25vl_lora(qwen25vl_lora_files):
|
def test_qwen25vl_lora(qwen25vl_lora_files):
|
||||||
"""Test Qwen 2.5 VL model with LoRA"""
|
"""Test Qwen 2.5 VL model with LoRA"""
|
||||||
config = TestConfig(model_path=QWEN25VL_MODEL_PATH, lora_path=qwen25vl_lora_files)
|
config = TestConfig(model_path=QWEN25VL_MODEL_PATH, lora_path=qwen25vl_lora_files)
|
||||||
tester = Qwen2VLTester(config)
|
with Qwen2VLTester(config) as tester:
|
||||||
|
# Test with different LoRA IDs
|
||||||
# Test with different LoRA IDs
|
for lora_id in [1, 2]:
|
||||||
for lora_id in [1, 2]:
|
tester.run_test(
|
||||||
tester.run_test(TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id)
|
TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(
|
@pytest.mark.skipif(
|
||||||
@@ -234,13 +241,13 @@ def test_qwen25vl_vision_lora(qwen25vl_vision_lora_files):
|
|||||||
mm_processor_cache_gb=0,
|
mm_processor_cache_gb=0,
|
||||||
enable_tower_connector_lora=True,
|
enable_tower_connector_lora=True,
|
||||||
)
|
)
|
||||||
tester = Qwen2VLTester(config)
|
with Qwen2VLTester(config) as tester:
|
||||||
for lora_id in [1, 2]:
|
for lora_id in [1, 2]:
|
||||||
tester.run_test(
|
tester.run_test(
|
||||||
TEST_IMAGES,
|
TEST_IMAGES,
|
||||||
expected_outputs=EXPECTED_OUTPUTS,
|
expected_outputs=EXPECTED_OUTPUTS,
|
||||||
lora_id=lora_id,
|
lora_id=lora_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_qwen3vl_vision_lora(qwen3vl_vision_lora_files):
|
def test_qwen3vl_vision_lora(qwen3vl_vision_lora_files):
|
||||||
@@ -253,13 +260,13 @@ def test_qwen3vl_vision_lora(qwen3vl_vision_lora_files):
|
|||||||
mm_processor_cache_gb=0,
|
mm_processor_cache_gb=0,
|
||||||
enable_tower_connector_lora=True,
|
enable_tower_connector_lora=True,
|
||||||
)
|
)
|
||||||
tester = Qwen2VLTester(config)
|
with Qwen2VLTester(config) as tester:
|
||||||
for lora_id in [1, 2]:
|
for lora_id in [1, 2]:
|
||||||
tester.run_test(
|
tester.run_test(
|
||||||
TEST_IMAGES,
|
TEST_IMAGES,
|
||||||
expected_outputs=EXPECTED_OUTPUTS,
|
expected_outputs=EXPECTED_OUTPUTS,
|
||||||
lora_id=lora_id,
|
lora_id=lora_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_qwen2vl_multiple_lora_types(
|
def test_qwen2vl_multiple_lora_types(
|
||||||
@@ -287,34 +294,33 @@ def test_qwen2vl_multiple_lora_types(
|
|||||||
mm_processor_cache_gb=0,
|
mm_processor_cache_gb=0,
|
||||||
enable_tower_connector_lora=True,
|
enable_tower_connector_lora=True,
|
||||||
)
|
)
|
||||||
tester = Qwen2VLTester(config)
|
with Qwen2VLTester(config) as tester:
|
||||||
|
# Test 1: Language-only LoRA adapter
|
||||||
|
tester.config.lora_path = qwen2vl_language_lora_files
|
||||||
|
for lora_id in [1, 2]:
|
||||||
|
tester.run_test(
|
||||||
|
TEST_IMAGES,
|
||||||
|
expected_outputs=EXPECTED_OUTPUTS_LANGUAGE,
|
||||||
|
lora_id=lora_id,
|
||||||
|
lora_name="language_only",
|
||||||
|
)
|
||||||
|
|
||||||
# Test 1: Language-only LoRA adapter
|
# Test 2: Vision tower + connector LoRA adapter
|
||||||
tester.config.lora_path = qwen2vl_language_lora_files
|
tester.config.lora_path = qwen2vl_vision_tower_connector_lora_files
|
||||||
for lora_id in [1, 2]:
|
for lora_id in [3, 4]:
|
||||||
tester.run_test(
|
tester.run_test(
|
||||||
TEST_IMAGES,
|
TEST_IMAGES,
|
||||||
expected_outputs=EXPECTED_OUTPUTS_LANGUAGE,
|
expected_outputs=EXPECTED_OUTPUTS_VISION,
|
||||||
lora_id=lora_id,
|
lora_id=lora_id,
|
||||||
lora_name="language_only",
|
lora_name="vision_tower_connector",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Test 2: Vision tower + connector LoRA adapter
|
# Test 3: Vision tower only LoRA adapter (no connector)
|
||||||
tester.config.lora_path = qwen2vl_vision_tower_connector_lora_files
|
tester.config.lora_path = qwen2vl_vision_tower_lora_files
|
||||||
for lora_id in [3, 4]:
|
for lora_id in [5, 6]:
|
||||||
tester.run_test(
|
tester.run_test(
|
||||||
TEST_IMAGES,
|
TEST_IMAGES,
|
||||||
expected_outputs=EXPECTED_OUTPUTS_VISION,
|
expected_outputs=EXPECTED_OUTPUTS_VISION_NO_CONNECTOR,
|
||||||
lora_id=lora_id,
|
lora_id=lora_id,
|
||||||
lora_name="vision_tower_connector",
|
lora_name="vision_tower",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Test 3: Vision tower only LoRA adapter (no connector)
|
|
||||||
tester.config.lora_path = qwen2vl_vision_tower_lora_files
|
|
||||||
for lora_id in [5, 6]:
|
|
||||||
tester.run_test(
|
|
||||||
TEST_IMAGES,
|
|
||||||
expected_outputs=EXPECTED_OUTPUTS_VISION_NO_CONNECTOR,
|
|
||||||
lora_id=lora_id,
|
|
||||||
lora_name="vision_tower",
|
|
||||||
)
|
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
|
|||||||
SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "")
|
SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "")
|
||||||
|
|
||||||
# Trap the SIGINT signal (triggered by Ctrl+C)
|
# Trap the SIGINT signal (triggered by Ctrl+C)
|
||||||
trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
|
trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT
|
||||||
|
|
||||||
# Waits for vLLM to start.
|
# Waits for vLLM to start.
|
||||||
wait_for_server() {
|
wait_for_server() {
|
||||||
@@ -110,10 +110,20 @@ wait_for_server() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Function to clean up previous instances
|
# Function to clean up previous instances
|
||||||
|
wait_for_gpu_memory_release() {
|
||||||
|
if [[ "$SMI_BIN" == *"rocm"* ]]; then
|
||||||
|
PYTHONPATH="${GIT_ROOT}" python3 -c "from tests.utils import wait_for_rocm_memory_to_settle; wait_for_rocm_memory_to_settle()"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
cleanup_instances() {
|
cleanup_instances() {
|
||||||
echo "Cleaning up any running vLLM instances..."
|
echo "Cleaning up any running vLLM instances..."
|
||||||
pkill -f "vllm serve" || true
|
pkill -f "toy_proxy_server.py" || true
|
||||||
|
pkill -TERM -f "vllm serve" || true
|
||||||
|
sleep 3
|
||||||
|
pkill -9 -f "vllm serve" || true
|
||||||
sleep 2
|
sleep 2
|
||||||
|
wait_for_gpu_memory_release
|
||||||
}
|
}
|
||||||
|
|
||||||
get_num_gpus() {
|
get_num_gpus() {
|
||||||
@@ -132,6 +142,7 @@ get_num_gpus() {
|
|||||||
# Function to run tests for a specific model
|
# Function to run tests for a specific model
|
||||||
run_tests_for_model() {
|
run_tests_for_model() {
|
||||||
local model_name=$1
|
local model_name=$1
|
||||||
|
cleanup_instances
|
||||||
echo "================================"
|
echo "================================"
|
||||||
echo "Testing model: $model_name"
|
echo "Testing model: $model_name"
|
||||||
echo "================================"
|
echo "================================"
|
||||||
@@ -303,7 +314,6 @@ run_tests_for_model() {
|
|||||||
|
|
||||||
# Clean up before running next model
|
# Clean up before running next model
|
||||||
cleanup_instances
|
cleanup_instances
|
||||||
sleep 3
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Run tests for each model
|
# Run tests for each model
|
||||||
|
|||||||
Reference in New Issue
Block a user