[ROCm][CI] Solidify entrypoint LLM lifecycle (#51280)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-08-13 12:24:02 -05:00
committed by GitHub
parent 69d4c3a06b
commit b96bcd0b47
24 changed files with 1002 additions and 1103 deletions
@@ -9,9 +9,6 @@ import pytest
import regex as re
import urllib3
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_CONFIGS = [
{
"model": "facebook/opt-125m",
@@ -55,19 +52,26 @@ MODEL_CONFIGS = [
]
def _create_runner(vllm_runner, model_config):
runner_config = model_config.copy()
model = runner_config.pop("model")
tokenizer_name = runner_config.pop("tokenizer", None)
return vllm_runner(model, tokenizer_name=tokenizer_name, **runner_config)
@pytest.fixture(scope="module")
def cache_models():
def cache_models(vllm_runner):
# Cache model files first
for model_config in MODEL_CONFIGS:
LLM(**model_config)
cleanup_dist_env_and_memory()
with _create_runner(vllm_runner, model_config):
pass
yield
@pytest.mark.skip_global_cleanup
@pytest.mark.usefixtures("cache_models")
def test_offline_mode(monkeypatch: pytest.MonkeyPatch):
def test_offline_mode(monkeypatch: pytest.MonkeyPatch, vllm_runner):
# Set HF to offline mode and ensure we can still construct an LLM
with monkeypatch.context() as m:
try:
@@ -93,7 +97,8 @@ def test_offline_mode(monkeypatch: pytest.MonkeyPatch):
_re_import_modules()
# Cached model files should be used in offline mode
for model_config in MODEL_CONFIGS:
LLM(**model_config)
with _create_runner(vllm_runner, model_config):
pass
finally:
# Reset the environment after the test
# NB: Assuming tests are run in online mode
@@ -136,7 +141,7 @@ def _re_import_modules():
@pytest.mark.skip_global_cleanup
@pytest.mark.usefixtures("cache_models")
def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch):
def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch, vllm_runner):
# Set HF to offline mode and ensure we can still construct an LLM
with monkeypatch.context() as m:
try:
@@ -159,7 +164,8 @@ def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch):
# Need to re-import huggingface_hub
# and friends to set up offline mode
_re_import_modules()
LLM(model="facebook/opt-125m")
with vllm_runner("facebook/opt-125m"):
pass
finally:
# Reset the environment after the test
# NB: Assuming tests are run in online mode
+21 -36
View File
@@ -4,46 +4,36 @@ import weakref
import pytest
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import SamplingParams
@pytest.fixture(scope="function")
def text_llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct", enforce_eager=True, seed=0)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
def text_llm(vllm_runner):
with vllm_runner(
"meta-llama/Llama-3.2-1B-Instruct", enforce_eager=True, seed=0
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.fixture(scope="function")
def llm_for_failure_test():
def llm_for_failure_test(vllm_runner):
"""
Fixture for testing issue #26081.
Uses a small max_model_len to easily trigger length errors.
"""
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model="meta-llama/Llama-3.2-1B-Instruct",
with vllm_runner(
"meta-llama/Llama-3.2-1B-Instruct",
enforce_eager=True,
seed=0,
max_model_len=128,
disable_log_stats=True,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
def test_chat(text_llm):
@@ -99,21 +89,16 @@ def test_llm_chat_tokenization_no_double_bos(text_llm):
@pytest.fixture(scope="function")
def thinking_llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model="Qwen/Qwen3-0.6B",
def thinking_llm(vllm_runner):
with vllm_runner(
"Qwen/Qwen3-0.6B",
max_model_len=4096,
enforce_eager=True,
seed=0,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.parametrize("enable_thinking", [True, False])
+5 -7
View File
@@ -4,15 +4,13 @@
import pytest
import torch
from vllm import LLM
from ...utils import create_new_process_for_each_test
@pytest.mark.parametrize("tp_size", [1, 2])
@pytest.mark.parametrize("backend", ["mp", "ray"])
@create_new_process_for_each_test()
def test_collective_rpc(tp_size, backend, monkeypatch):
def test_collective_rpc(tp_size, backend, monkeypatch, vllm_runner):
if torch.accelerator.device_count() < tp_size:
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
if tp_size == 1 and backend == "ray":
@@ -26,11 +24,11 @@ def test_collective_rpc(tp_size, backend, monkeypatch):
return self.rank
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
llm = LLM(
model="hmellor/tiny-random-LlamaForCausalLM",
with vllm_runner(
"hmellor/tiny-random-LlamaForCausalLM",
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=tp_size,
distributed_executor_backend=backend,
)
assert llm.collective_rpc(echo_rank) == list(range(tp_size))
) as runner:
assert runner.llm.collective_rpc(echo_rank) == list(range(tp_size))
+28 -34
View File
@@ -6,7 +6,6 @@ import weakref
import pytest
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_NAME = "distilbert/distilgpt2"
@@ -26,22 +25,17 @@ TOKEN_IDS = [
@pytest.fixture(scope="module")
def llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model=MODEL_NAME,
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_num_batched_tokens=4096,
tensor_parallel_size=1,
gpu_memory_utilization=0.10,
enforce_eager=True,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
@@ -97,34 +91,34 @@ def test_single_prompt_priority(llm: LLM):
assert len(outputs) == 1
def test_max_model_len():
def test_max_model_len(vllm_runner):
max_model_len = 20
llm = LLM(
model=MODEL_NAME,
with vllm_runner(
MODEL_NAME,
max_model_len=max_model_len,
gpu_memory_utilization=0.10,
enforce_eager=True, # reduce test time
)
sampling_params = SamplingParams(max_tokens=max_model_len + 10)
outputs = llm.generate(PROMPTS, sampling_params)
for output in outputs:
num_total_tokens = len(output.prompt_token_ids) + len(
output.outputs[0].token_ids
)
# Total tokens must not exceed max_model_len.
# It can be less if generation finishes due to other reasons (e.g., EOS)
# before reaching the absolute model length limit.
assert num_total_tokens <= max_model_len
) as runner:
sampling_params = SamplingParams(max_tokens=max_model_len + 10)
outputs = runner.llm.generate(PROMPTS, sampling_params)
for output in outputs:
num_total_tokens = len(output.prompt_token_ids) + len(
output.outputs[0].token_ids
)
# Total tokens must not exceed max_model_len.
# It can be less if generation finishes due to other reasons (e.g., EOS)
# before reaching the absolute model length limit.
assert num_total_tokens <= max_model_len
def test_log_stats():
llm = LLM(
model=MODEL_NAME,
def test_log_stats(vllm_runner):
with vllm_runner(
MODEL_NAME,
disable_log_stats=False,
gpu_memory_utilization=0.10,
enforce_eager=True, # reduce test time
)
outputs = llm.generate(PROMPTS, sampling_params=None)
) as runner:
outputs = runner.llm.generate(PROMPTS, sampling_params=None)
# disable_log_stats is False, every output should have metrics
assert all(output.metrics is not None for output in outputs)
# disable_log_stats is False, every output should have metrics
assert all(output.metrics is not None for output in outputs)
+25 -12
View File
@@ -1,10 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm import LLM, SamplingParams
from vllm import SamplingParams
def test_gpu_memory_utilization():
def test_gpu_memory_utilization(vllm_runner):
prompts = [
"Hello, my name is",
"The president of the United States is",
@@ -15,13 +15,26 @@ def test_gpu_memory_utilization():
# makes sure gpu_memory_utilization is per-instance limit,
# not a global limit
llms = [
LLM(model="facebook/opt-125m", gpu_memory_utilization=0.3, enforce_eager=True)
for i in range(3)
]
for llm in llms:
outputs = llm.generate(prompts, sampling_params)
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
with (
vllm_runner(
"facebook/opt-125m",
gpu_memory_utilization=0.3,
enforce_eager=True,
) as runner_0,
vllm_runner(
"facebook/opt-125m",
gpu_memory_utilization=0.3,
enforce_eager=True,
) as runner_1,
vllm_runner(
"facebook/opt-125m",
gpu_memory_utilization=0.3,
enforce_eager=True,
) as runner_2,
):
for runner in (runner_0, runner_1, runner_2):
outputs = runner.llm.generate(prompts, sampling_params)
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+22 -17
View File
@@ -4,30 +4,35 @@
import pytest
import torch
from vllm import LLM
from vllm.exceptions import VLLMValidationError
def test_empty_prompt():
llm = LLM(model="openai-community/gpt2", enforce_eager=True)
with pytest.raises(VLLMValidationError, match="decoder prompt cannot be empty"):
llm.generate([""])
def test_empty_prompt(vllm_runner):
with (
vllm_runner("openai-community/gpt2", enforce_eager=True) as runner,
pytest.raises(VLLMValidationError, match="decoder prompt cannot be empty"),
):
runner.llm.generate([""])
def test_out_of_vocab_token():
llm = LLM(model="openai-community/gpt2", enforce_eager=True)
with pytest.raises(VLLMValidationError, match="out of vocabulary"):
llm.generate({"prompt_token_ids": [999999]})
def test_out_of_vocab_token(vllm_runner):
with (
vllm_runner("openai-community/gpt2", enforce_eager=True) as runner,
pytest.raises(VLLMValidationError, match="out of vocabulary"),
):
runner.llm.generate({"prompt_token_ids": [999999]})
def test_require_mm_embeds():
llm = LLM(
model="llava-hf/llava-1.5-7b-hf",
enforce_eager=True,
enable_mm_embeds=False,
)
with pytest.raises(ValueError, match="--enable-mm-embeds"):
llm.generate(
def test_require_mm_embeds(vllm_runner):
with (
vllm_runner(
"llava-hf/llava-1.5-7b-hf",
enforce_eager=True,
enable_mm_embeds=False,
) as runner,
pytest.raises(ValueError, match="--enable-mm-embeds"),
):
runner.llm.generate(
{
"prompt": "<image>",
"multi_modal_data": {"image": torch.empty(1, 1, 1)},
File diff suppressed because it is too large Load Diff