[ROCm][CI] Solidify entrypoint LLM lifecycle (#51280)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -6,7 +6,6 @@ import weakref
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, PoolingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
@@ -29,23 +28,20 @@ TOKEN_IDS = [
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -6,28 +6,26 @@ from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, PoolingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm import PoolingParams
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_seqs=2, # small to trigger tiling
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -7,7 +7,6 @@ import torch
|
||||
|
||||
from tests.models.utils import softmax
|
||||
from vllm import LLM, ClassificationRequestOutput, PoolingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
|
||||
@@ -18,23 +17,20 @@ num_labels = 2
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -7,7 +7,6 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import LLM, EmbeddingRequestOutput, PoolingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
@@ -18,23 +17,21 @@ embedding_size = 384
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
)
|
||||
assert embedding_size == llm.model_config.embedding_size
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
cleanup_dist_env_and_memory()
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
assert embedding_size == runner.llm.model_config.embedding_size
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -8,7 +8,6 @@ import torch
|
||||
|
||||
from tests.models.utils import softmax
|
||||
from vllm import LLM, PoolingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL_NAME = "internlm/internlm2-1_8b-reward"
|
||||
|
||||
@@ -16,24 +15,21 @@ prompts = ["The chef prepared a delicious meal."]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
seed=0,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -6,8 +6,6 @@ import weakref
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner
|
||||
from vllm import LLM
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
PROMPT = "The chef prepared a delicious meal."
|
||||
@@ -27,23 +25,20 @@ DTYPE = "half"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
||||
@@ -8,7 +8,6 @@ import torch
|
||||
|
||||
from tests.models.utils import softmax
|
||||
from vllm import LLM, PoolingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
|
||||
PROMPT = "The chef prepared a delicious meal."
|
||||
@@ -24,23 +23,20 @@ TEXTS_2 = [
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
||||
@@ -5,9 +5,6 @@ import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
from .util import ColBERTScoringHfRunner
|
||||
|
||||
MODEL_NAME = "answerdotai/answerai-colbert-small-v1"
|
||||
@@ -30,23 +27,20 @@ DTYPE = "half"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
||||
@@ -5,32 +5,26 @@ import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
from .util import make_base64_image, make_image_mm_param
|
||||
|
||||
MODEL_NAME = "vidore/colpali-v1.3-hf"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -6,7 +6,6 @@ import pytest
|
||||
|
||||
from vllm import LLM, PoolingRequestOutput
|
||||
from vllm.config import PoolerConfig
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
|
||||
@@ -17,24 +16,21 @@ num_labels = 2
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
pooler_config=PoolerConfig(task="token_classify"),
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -6,7 +6,6 @@ import pytest
|
||||
|
||||
from vllm import LLM, PoolingRequestOutput
|
||||
from vllm.config import PoolerConfig
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.tasks import PoolingTask
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
@@ -17,24 +16,22 @@ embedding_size = 384
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=None,
|
||||
pooler_config=PoolerConfig(task="token_embed"),
|
||||
max_num_batched_tokens=32768,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
)
|
||||
assert embedding_size == llm.model_config.embedding_size
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
cleanup_dist_env_and_memory()
|
||||
enable_chunked_prefill=None,
|
||||
) as runner:
|
||||
assert embedding_size == runner.llm.model_config.embedding_size
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
|
||||
Reference in New Issue
Block a user