[Bugfix][V1] Clean up compiled-model bytecode hooks on VllmRunner exit (#45195)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
14b438a98b
commit
7b5d60cc37
@@ -1294,6 +1294,7 @@ class VllmRunner:
|
||||
# Ignore shutdown errors as cleanup will still proceed
|
||||
pass
|
||||
del self.llm
|
||||
torch._dynamo.reset()
|
||||
cleanup_dist_env_and_memory()
|
||||
self._wait_for_rocm_memory_release(gpu_memory_utilization)
|
||||
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import wait_for_gpu_memory_to_clear
|
||||
from tests.conftest import VllmRunner
|
||||
from tests.utils import create_new_process_for_each_test, wait_for_gpu_memory_to_clear
|
||||
from tests.v1.shutdown.utils import (
|
||||
SHUTDOWN_TEST_THRESHOLD_BYTES,
|
||||
SHUTDOWN_TEST_TIMEOUT_SEC,
|
||||
@@ -106,3 +107,29 @@ def test_llm_delete(
|
||||
devices=list(range(tensor_parallel_size)),
|
||||
threshold_bytes=SHUTDOWN_TEST_THRESHOLD_BYTES,
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
@pytest.mark.timeout(SHUTDOWN_TEST_TIMEOUT_SEC)
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("send_one_request", [False, True])
|
||||
def test_llm_delete_inprocess(
|
||||
monkeypatch,
|
||||
model: str,
|
||||
send_one_request: bool,
|
||||
) -> None:
|
||||
"""Test that VllmRunner frees GPU memory in in-process (no MP) mode."""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
with VllmRunner(model) as vllm_model:
|
||||
if send_one_request:
|
||||
vllm_model.generate(
|
||||
["Hello my name is"],
|
||||
SamplingParams(max_tokens=1),
|
||||
)
|
||||
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=[0],
|
||||
threshold_bytes=SHUTDOWN_TEST_THRESHOLD_BYTES,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user