[ROCm][CI] Solidify entrypoint LLM lifecycle (#51280)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -9,9 +9,6 @@ import pytest
|
||||
import regex as re
|
||||
import urllib3
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL_CONFIGS = [
|
||||
{
|
||||
"model": "facebook/opt-125m",
|
||||
@@ -55,19 +52,26 @@ MODEL_CONFIGS = [
|
||||
]
|
||||
|
||||
|
||||
def _create_runner(vllm_runner, model_config):
|
||||
runner_config = model_config.copy()
|
||||
model = runner_config.pop("model")
|
||||
tokenizer_name = runner_config.pop("tokenizer", None)
|
||||
return vllm_runner(model, tokenizer_name=tokenizer_name, **runner_config)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def cache_models():
|
||||
def cache_models(vllm_runner):
|
||||
# Cache model files first
|
||||
for model_config in MODEL_CONFIGS:
|
||||
LLM(**model_config)
|
||||
cleanup_dist_env_and_memory()
|
||||
with _create_runner(vllm_runner, model_config):
|
||||
pass
|
||||
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.usefixtures("cache_models")
|
||||
def test_offline_mode(monkeypatch: pytest.MonkeyPatch):
|
||||
def test_offline_mode(monkeypatch: pytest.MonkeyPatch, vllm_runner):
|
||||
# Set HF to offline mode and ensure we can still construct an LLM
|
||||
with monkeypatch.context() as m:
|
||||
try:
|
||||
@@ -93,7 +97,8 @@ def test_offline_mode(monkeypatch: pytest.MonkeyPatch):
|
||||
_re_import_modules()
|
||||
# Cached model files should be used in offline mode
|
||||
for model_config in MODEL_CONFIGS:
|
||||
LLM(**model_config)
|
||||
with _create_runner(vllm_runner, model_config):
|
||||
pass
|
||||
finally:
|
||||
# Reset the environment after the test
|
||||
# NB: Assuming tests are run in online mode
|
||||
@@ -136,7 +141,7 @@ def _re_import_modules():
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.usefixtures("cache_models")
|
||||
def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch):
|
||||
def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch, vllm_runner):
|
||||
# Set HF to offline mode and ensure we can still construct an LLM
|
||||
with monkeypatch.context() as m:
|
||||
try:
|
||||
@@ -159,7 +164,8 @@ def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch):
|
||||
# Need to re-import huggingface_hub
|
||||
# and friends to set up offline mode
|
||||
_re_import_modules()
|
||||
LLM(model="facebook/opt-125m")
|
||||
with vllm_runner("facebook/opt-125m"):
|
||||
pass
|
||||
finally:
|
||||
# Reset the environment after the test
|
||||
# NB: Assuming tests are run in online mode
|
||||
|
||||
@@ -4,46 +4,36 @@ import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def text_llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct", enforce_eager=True, seed=0)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
def text_llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
"meta-llama/Llama-3.2-1B-Instruct", enforce_eager=True, seed=0
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def llm_for_failure_test():
|
||||
def llm_for_failure_test(vllm_runner):
|
||||
"""
|
||||
Fixture for testing issue #26081.
|
||||
Uses a small max_model_len to easily trigger length errors.
|
||||
"""
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model="meta-llama/Llama-3.2-1B-Instruct",
|
||||
with vllm_runner(
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
max_model_len=128,
|
||||
disable_log_stats=True,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
def test_chat(text_llm):
|
||||
@@ -99,21 +89,16 @@ def test_llm_chat_tokenization_no_double_bos(text_llm):
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def thinking_llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model="Qwen/Qwen3-0.6B",
|
||||
def thinking_llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
max_model_len=4096,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_thinking", [True, False])
|
||||
|
||||
@@ -4,15 +4,13 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import LLM
|
||||
|
||||
from ...utils import create_new_process_for_each_test
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
@pytest.mark.parametrize("backend", ["mp", "ray"])
|
||||
@create_new_process_for_each_test()
|
||||
def test_collective_rpc(tp_size, backend, monkeypatch):
|
||||
def test_collective_rpc(tp_size, backend, monkeypatch, vllm_runner):
|
||||
if torch.accelerator.device_count() < tp_size:
|
||||
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
|
||||
if tp_size == 1 and backend == "ray":
|
||||
@@ -26,11 +24,11 @@ def test_collective_rpc(tp_size, backend, monkeypatch):
|
||||
return self.rank
|
||||
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
llm = LLM(
|
||||
model="hmellor/tiny-random-LlamaForCausalLM",
|
||||
with vllm_runner(
|
||||
"hmellor/tiny-random-LlamaForCausalLM",
|
||||
enforce_eager=True,
|
||||
load_format="dummy",
|
||||
tensor_parallel_size=tp_size,
|
||||
distributed_executor_backend=backend,
|
||||
)
|
||||
assert llm.collective_rpc(echo_rank) == list(range(tp_size))
|
||||
) as runner:
|
||||
assert runner.llm.collective_rpc(echo_rank) == list(range(tp_size))
|
||||
|
||||
@@ -6,7 +6,6 @@ import weakref
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL_NAME = "distilbert/distilgpt2"
|
||||
|
||||
@@ -26,22 +25,17 @@ TOKEN_IDS = [
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def llm(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_num_batched_tokens=4096,
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.10,
|
||||
enforce_eager=True,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
) as runner:
|
||||
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||
yield weakref.proxy(runner.llm)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@@ -97,34 +91,34 @@ def test_single_prompt_priority(llm: LLM):
|
||||
assert len(outputs) == 1
|
||||
|
||||
|
||||
def test_max_model_len():
|
||||
def test_max_model_len(vllm_runner):
|
||||
max_model_len = 20
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
max_model_len=max_model_len,
|
||||
gpu_memory_utilization=0.10,
|
||||
enforce_eager=True, # reduce test time
|
||||
)
|
||||
sampling_params = SamplingParams(max_tokens=max_model_len + 10)
|
||||
outputs = llm.generate(PROMPTS, sampling_params)
|
||||
for output in outputs:
|
||||
num_total_tokens = len(output.prompt_token_ids) + len(
|
||||
output.outputs[0].token_ids
|
||||
)
|
||||
# Total tokens must not exceed max_model_len.
|
||||
# It can be less if generation finishes due to other reasons (e.g., EOS)
|
||||
# before reaching the absolute model length limit.
|
||||
assert num_total_tokens <= max_model_len
|
||||
) as runner:
|
||||
sampling_params = SamplingParams(max_tokens=max_model_len + 10)
|
||||
outputs = runner.llm.generate(PROMPTS, sampling_params)
|
||||
for output in outputs:
|
||||
num_total_tokens = len(output.prompt_token_ids) + len(
|
||||
output.outputs[0].token_ids
|
||||
)
|
||||
# Total tokens must not exceed max_model_len.
|
||||
# It can be less if generation finishes due to other reasons (e.g., EOS)
|
||||
# before reaching the absolute model length limit.
|
||||
assert num_total_tokens <= max_model_len
|
||||
|
||||
|
||||
def test_log_stats():
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
def test_log_stats(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
disable_log_stats=False,
|
||||
gpu_memory_utilization=0.10,
|
||||
enforce_eager=True, # reduce test time
|
||||
)
|
||||
outputs = llm.generate(PROMPTS, sampling_params=None)
|
||||
) as runner:
|
||||
outputs = runner.llm.generate(PROMPTS, sampling_params=None)
|
||||
|
||||
# disable_log_stats is False, every output should have metrics
|
||||
assert all(output.metrics is not None for output in outputs)
|
||||
# disable_log_stats is False, every output should have metrics
|
||||
assert all(output.metrics is not None for output in outputs)
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm import SamplingParams
|
||||
|
||||
|
||||
def test_gpu_memory_utilization():
|
||||
def test_gpu_memory_utilization(vllm_runner):
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
@@ -15,13 +15,26 @@ def test_gpu_memory_utilization():
|
||||
|
||||
# makes sure gpu_memory_utilization is per-instance limit,
|
||||
# not a global limit
|
||||
llms = [
|
||||
LLM(model="facebook/opt-125m", gpu_memory_utilization=0.3, enforce_eager=True)
|
||||
for i in range(3)
|
||||
]
|
||||
for llm in llms:
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
|
||||
with (
|
||||
vllm_runner(
|
||||
"facebook/opt-125m",
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
) as runner_0,
|
||||
vllm_runner(
|
||||
"facebook/opt-125m",
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
) as runner_1,
|
||||
vllm_runner(
|
||||
"facebook/opt-125m",
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
) as runner_2,
|
||||
):
|
||||
for runner in (runner_0, runner_1, runner_2):
|
||||
outputs = runner.llm.generate(prompts, sampling_params)
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
|
||||
|
||||
@@ -4,30 +4,35 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
|
||||
def test_empty_prompt():
|
||||
llm = LLM(model="openai-community/gpt2", enforce_eager=True)
|
||||
with pytest.raises(VLLMValidationError, match="decoder prompt cannot be empty"):
|
||||
llm.generate([""])
|
||||
def test_empty_prompt(vllm_runner):
|
||||
with (
|
||||
vllm_runner("openai-community/gpt2", enforce_eager=True) as runner,
|
||||
pytest.raises(VLLMValidationError, match="decoder prompt cannot be empty"),
|
||||
):
|
||||
runner.llm.generate([""])
|
||||
|
||||
|
||||
def test_out_of_vocab_token():
|
||||
llm = LLM(model="openai-community/gpt2", enforce_eager=True)
|
||||
with pytest.raises(VLLMValidationError, match="out of vocabulary"):
|
||||
llm.generate({"prompt_token_ids": [999999]})
|
||||
def test_out_of_vocab_token(vllm_runner):
|
||||
with (
|
||||
vllm_runner("openai-community/gpt2", enforce_eager=True) as runner,
|
||||
pytest.raises(VLLMValidationError, match="out of vocabulary"),
|
||||
):
|
||||
runner.llm.generate({"prompt_token_ids": [999999]})
|
||||
|
||||
|
||||
def test_require_mm_embeds():
|
||||
llm = LLM(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
enforce_eager=True,
|
||||
enable_mm_embeds=False,
|
||||
)
|
||||
with pytest.raises(ValueError, match="--enable-mm-embeds"):
|
||||
llm.generate(
|
||||
def test_require_mm_embeds(vllm_runner):
|
||||
with (
|
||||
vllm_runner(
|
||||
"llava-hf/llava-1.5-7b-hf",
|
||||
enforce_eager=True,
|
||||
enable_mm_embeds=False,
|
||||
) as runner,
|
||||
pytest.raises(ValueError, match="--enable-mm-embeds"),
|
||||
):
|
||||
runner.llm.generate(
|
||||
{
|
||||
"prompt": "<image>",
|
||||
"multi_modal_data": {"image": torch.empty(1, 1, 1)},
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user