iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[ROCm][CI] Solidify entrypoint LLM lifecycle (#51280)

Browse Source 
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-08-13 12:24:02 -05:00
committed by GitHub
parent 69d4c3a06b
commit b96bcd0b47
24 changed files with 1002 additions and 1103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/entrypoints/llm/offline_mode/test_offline_mode.py
+16 -10
View File
@@ -9,9 +9,6 @@ import pytest
import regex as re
import urllib3
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_CONFIGS = [
{
"model": "facebook/opt-125m",
@@ -55,19 +52,26 @@ MODEL_CONFIGS = [
]
def _create_runner(vllm_runner, model_config):
runner_config = model_config.copy()
model = runner_config.pop("model")
tokenizer_name = runner_config.pop("tokenizer", None)
return vllm_runner(model, tokenizer_name=tokenizer_name, **runner_config)
@pytest.fixture(scope="module")
def cache_models():
def cache_models(vllm_runner):
# Cache model files first
for model_config in MODEL_CONFIGS:
LLM(**model_config)
cleanup_dist_env_and_memory()
with _create_runner(vllm_runner, model_config):
pass
yield
@pytest.mark.skip_global_cleanup
@pytest.mark.usefixtures("cache_models")
def test_offline_mode(monkeypatch: pytest.MonkeyPatch):
def test_offline_mode(monkeypatch: pytest.MonkeyPatch, vllm_runner):
# Set HF to offline mode and ensure we can still construct an LLM
with monkeypatch.context() as m:
try:
@@ -93,7 +97,8 @@ def test_offline_mode(monkeypatch: pytest.MonkeyPatch):
_re_import_modules()
# Cached model files should be used in offline mode
for model_config in MODEL_CONFIGS:
LLM(**model_config)
with _create_runner(vllm_runner, model_config):
pass
finally:
# Reset the environment after the test
# NB: Assuming tests are run in online mode
@@ -136,7 +141,7 @@ def _re_import_modules():
@pytest.mark.skip_global_cleanup
@pytest.mark.usefixtures("cache_models")
def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch):
def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch, vllm_runner):
# Set HF to offline mode and ensure we can still construct an LLM
with monkeypatch.context() as m:
try:
@@ -159,7 +164,8 @@ def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch):
# Need to re-import huggingface_hub
# and friends to set up offline mode
_re_import_modules()
LLM(model="facebook/opt-125m")
with vllm_runner("facebook/opt-125m"):
pass
finally:
# Reset the environment after the test
# NB: Assuming tests are run in online mode
tests/entrypoints/llm/test_chat.py
+21 -36
View File
@@ -4,46 +4,36 @@ import weakref
import pytest
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import SamplingParams
@pytest.fixture(scope="function")
def text_llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct", enforce_eager=True, seed=0)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
def text_llm(vllm_runner):
with vllm_runner(
"meta-llama/Llama-3.2-1B-Instruct", enforce_eager=True, seed=0
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.fixture(scope="function")
def llm_for_failure_test():
def llm_for_failure_test(vllm_runner):
"""
Fixture for testing issue #26081.
Uses a small max_model_len to easily trigger length errors.
"""
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model="meta-llama/Llama-3.2-1B-Instruct",
with vllm_runner(
"meta-llama/Llama-3.2-1B-Instruct",
enforce_eager=True,
seed=0,
max_model_len=128,
disable_log_stats=True,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
def test_chat(text_llm):
@@ -99,21 +89,16 @@ def test_llm_chat_tokenization_no_double_bos(text_llm):
@pytest.fixture(scope="function")
def thinking_llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model="Qwen/Qwen3-0.6B",
def thinking_llm(vllm_runner):
with vllm_runner(
"Qwen/Qwen3-0.6B",
max_model_len=4096,
enforce_eager=True,
seed=0,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.parametrize("enable_thinking", [True, False])
tests/entrypoints/llm/test_collective_rpc.py
+5 -7
View File
@@ -4,15 +4,13 @@
import pytest
import torch
from vllm import LLM
from ...utils import create_new_process_for_each_test
@pytest.mark.parametrize("tp_size", [1, 2])
@pytest.mark.parametrize("backend", ["mp", "ray"])
@create_new_process_for_each_test()
def test_collective_rpc(tp_size, backend, monkeypatch):
def test_collective_rpc(tp_size, backend, monkeypatch, vllm_runner):
if torch.accelerator.device_count() < tp_size:
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
if tp_size == 1 and backend == "ray":
@@ -26,11 +24,11 @@ def test_collective_rpc(tp_size, backend, monkeypatch):
return self.rank
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
llm = LLM(
model="hmellor/tiny-random-LlamaForCausalLM",
with vllm_runner(
"hmellor/tiny-random-LlamaForCausalLM",
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=tp_size,
distributed_executor_backend=backend,
)
assert llm.collective_rpc(echo_rank) == list(range(tp_size))
) as runner:
assert runner.llm.collective_rpc(echo_rank) == list(range(tp_size))
tests/entrypoints/llm/test_generate.py
+28 -34
View File
@@ -6,7 +6,6 @@ import weakref
import pytest
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_NAME = "distilbert/distilgpt2"
@@ -26,22 +25,17 @@ TOKEN_IDS = [
@pytest.fixture(scope="module")
def llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model=MODEL_NAME,
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_num_batched_tokens=4096,
tensor_parallel_size=1,
gpu_memory_utilization=0.10,
enforce_eager=True,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
@@ -97,34 +91,34 @@ def test_single_prompt_priority(llm: LLM):
assert len(outputs) == 1
def test_max_model_len():
def test_max_model_len(vllm_runner):
max_model_len = 20
llm = LLM(
model=MODEL_NAME,
with vllm_runner(
MODEL_NAME,
max_model_len=max_model_len,
gpu_memory_utilization=0.10,
enforce_eager=True, # reduce test time
)
sampling_params = SamplingParams(max_tokens=max_model_len + 10)
outputs = llm.generate(PROMPTS, sampling_params)
for output in outputs:
num_total_tokens = len(output.prompt_token_ids) + len(
output.outputs[0].token_ids
)
# Total tokens must not exceed max_model_len.
# It can be less if generation finishes due to other reasons (e.g., EOS)
# before reaching the absolute model length limit.
assert num_total_tokens <= max_model_len
) as runner:
sampling_params = SamplingParams(max_tokens=max_model_len + 10)
outputs = runner.llm.generate(PROMPTS, sampling_params)
for output in outputs:
num_total_tokens = len(output.prompt_token_ids) + len(
output.outputs[0].token_ids
)
# Total tokens must not exceed max_model_len.
# It can be less if generation finishes due to other reasons (e.g., EOS)
# before reaching the absolute model length limit.
assert num_total_tokens <= max_model_len
def test_log_stats():
llm = LLM(
model=MODEL_NAME,
def test_log_stats(vllm_runner):
with vllm_runner(
MODEL_NAME,
disable_log_stats=False,
gpu_memory_utilization=0.10,
enforce_eager=True, # reduce test time
)
outputs = llm.generate(PROMPTS, sampling_params=None)
) as runner:
outputs = runner.llm.generate(PROMPTS, sampling_params=None)
# disable_log_stats is False, every output should have metrics
assert all(output.metrics is not None for output in outputs)
# disable_log_stats is False, every output should have metrics
assert all(output.metrics is not None for output in outputs)
tests/entrypoints/llm/test_gpu_utilization.py
+25 -12
View File
@@ -1,10 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm import LLM, SamplingParams
from vllm import SamplingParams
def test_gpu_memory_utilization():
def test_gpu_memory_utilization(vllm_runner):
prompts = [
"Hello, my name is",
"The president of the United States is",
@@ -15,13 +15,26 @@ def test_gpu_memory_utilization():
# makes sure gpu_memory_utilization is per-instance limit,
# not a global limit
llms = [
LLM(model="facebook/opt-125m", gpu_memory_utilization=0.3, enforce_eager=True)
for i in range(3)
]
for llm in llms:
outputs = llm.generate(prompts, sampling_params)
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
with (
vllm_runner(
"facebook/opt-125m",
gpu_memory_utilization=0.3,
enforce_eager=True,
) as runner_0,
vllm_runner(
"facebook/opt-125m",
gpu_memory_utilization=0.3,
enforce_eager=True,
) as runner_1,
vllm_runner(
"facebook/opt-125m",
gpu_memory_utilization=0.3,
enforce_eager=True,
) as runner_2,
):
for runner in (runner_0, runner_1, runner_2):
outputs = runner.llm.generate(prompts, sampling_params)
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
tests/entrypoints/llm/test_prompt_validation.py
+22 -17
View File
@@ -4,30 +4,35 @@
import pytest
import torch
from vllm import LLM
from vllm.exceptions import VLLMValidationError
def test_empty_prompt():
llm = LLM(model="openai-community/gpt2", enforce_eager=True)
with pytest.raises(VLLMValidationError, match="decoder prompt cannot be empty"):
llm.generate([""])
def test_empty_prompt(vllm_runner):
with (
vllm_runner("openai-community/gpt2", enforce_eager=True) as runner,
pytest.raises(VLLMValidationError, match="decoder prompt cannot be empty"),
):
runner.llm.generate([""])
def test_out_of_vocab_token():
llm = LLM(model="openai-community/gpt2", enforce_eager=True)
with pytest.raises(VLLMValidationError, match="out of vocabulary"):
llm.generate({"prompt_token_ids": [999999]})
def test_out_of_vocab_token(vllm_runner):
with (
vllm_runner("openai-community/gpt2", enforce_eager=True) as runner,
pytest.raises(VLLMValidationError, match="out of vocabulary"),
):
runner.llm.generate({"prompt_token_ids": [999999]})
def test_require_mm_embeds():
llm = LLM(
model="llava-hf/llava-1.5-7b-hf",
enforce_eager=True,
enable_mm_embeds=False,
)
with pytest.raises(ValueError, match="--enable-mm-embeds"):
llm.generate(
def test_require_mm_embeds(vllm_runner):
with (
vllm_runner(
"llava-hf/llava-1.5-7b-hf",
enforce_eager=True,
enable_mm_embeds=False,
) as runner,
pytest.raises(ValueError, match="--enable-mm-embeds"),
):
runner.llm.generate(
{
"prompt": "<image>",
"multi_modal_data": {"image": torch.empty(1, 1, 1)},
tests/entrypoints/llm/test_struct_output_generate.py
+619 -615
View File
File diff suppressed because it is too large Load Diff
tests/entrypoints/multimodal/conftest.py
-73
View File
@@ -1,11 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from typing import Any
import pytest
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
TEST_IMAGE_ASSETS = [
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
@@ -13,70 +7,3 @@ TEST_IMAGE_ASSETS = [
"1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
"RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
]
def _shutdown_llm(llm: Any, gpu_memory_utilization: float) -> None:
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.platforms import current_platform
try:
shutdown_timeout = 60.0 if current_platform.is_rocm() else None
llm.llm_engine.engine_core.shutdown(timeout=shutdown_timeout)
except Exception:
pass
del llm
try:
import torch
torch._dynamo.reset()
except Exception:
pass
cleanup_dist_env_and_memory()
if current_platform.is_rocm():
from tests.utils import wait_for_rocm_memory_to_settle
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
@contextmanager
def managed_llm(*args: Any, **kwargs: Any) -> Iterator[Any]:
from vllm import LLM
llm = LLM(*args, **kwargs)
gpu_memory_utilization = (
llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
)
try:
yield llm
finally:
_shutdown_llm(llm, gpu_memory_utilization)
def _make_managed_llm_factory() -> Iterator[Callable[..., Any]]:
from vllm import LLM
llms: list[tuple[Any, float]] = []
def make_llm(*args: Any, **kwargs: Any) -> Any:
llm = LLM(*args, **kwargs)
gpu_memory_utilization = (
llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
)
llms.append((llm, gpu_memory_utilization))
return llm
try:
yield make_llm
finally:
while llms:
llm, gpu_memory_utilization = llms.pop()
_shutdown_llm(llm, gpu_memory_utilization)
@pytest.fixture
def multimodal_llm_factory() -> Iterator[Callable[..., Any]]:
yield from _make_managed_llm_factory()
tests/entrypoints/multimodal/llm/test_chat.py
+9 -4
View File
@@ -1,21 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
@pytest.fixture(scope="function")
def vision_llm(multimodal_llm_factory):
return multimodal_llm_factory(
model="microsoft/Phi-3.5-vision-instruct",
def vision_llm(vllm_runner):
with vllm_runner(
"microsoft/Phi-3.5-vision-instruct",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
trust_remote_code=True,
limit_mm_per_prompt={"image": 2},
seed=0,
)
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.parametrize(
tests/entrypoints/multimodal/llm/test_mm_cache_external_injection.py
+77 -79
View File
@@ -69,7 +69,7 @@ def test_inject_into_mm_cache(
image_urls,
mm_processor_cache_type,
caplog_vllm,
multimodal_llm_factory,
vllm_runner,
):
"""Test that inject_into_mm_cache() injects pre-processed mm_kwargs into
the processor cache and MM cache hit metrics are updated correctly.
@@ -79,117 +79,115 @@ def test_inject_into_mm_cache(
2. Extract cached kwargs, call inject_into_mm_cache with a new hash,
then generate with a pre-rendered input -> verifies injection works
"""
llm = multimodal_llm_factory(
model="llava-hf/llava-1.5-7b-hf",
with vllm_runner(
"llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
mm_processor_cache_type=mm_processor_cache_type,
)
) as runner:
# Step 1: Normal requests to populate the cache
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
# Step 1: Normal requests to populate the cache
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(50.0)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(50.0)
# Step 2: Use a second image to get valid expanded tokens and
# placeholder positions via the renderer.
runner.llm.chat(_make_messages(image_urls[1]))
queries_before = _get_mm_cache_stats(runner.llm.get_metrics())[0] # 3
# Step 2: Use a second image to get valid expanded tokens and
# placeholder positions via the renderer.
llm.chat(_make_messages(image_urls[1]))
queries_before = _get_mm_cache_stats(llm.get_metrics())[0] # 3
renderer = runner.llm.llm_engine.renderer
cache = renderer.mm_processor_cache
assert cache is not None, "Processor cache should be enabled"
renderer = llm.llm_engine.renderer
cache = renderer.mm_processor_cache
assert cache is not None, "Processor cache should be enabled"
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[1])],
ChatParams(),
)
eng_input = eng_prompts[0]
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[1])],
ChatParams(),
)
eng_input = eng_prompts[0]
# Inject pre-processed mm_kwargs with a NEW hash via public API
new_mm_hash = "deadbeef" * 8
mm_hashes = {"image": [new_mm_hash]}
mm_kwargs = eng_input["mm_kwargs"]
# Inject pre-processed mm_kwargs with a NEW hash via public API
new_mm_hash = "deadbeef" * 8
mm_hashes = {"image": [new_mm_hash]}
mm_kwargs = eng_input["mm_kwargs"]
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# Build pre-rendered input (no externally_processed flag needed)
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
# Build pre-rendered input (no externally_processed flag needed)
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
runner.llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
# Verify cache was queried and injection happened
queries_after = _get_mm_cache_stats(llm.get_metrics())[0]
assert queries_after > queries_before, (
"Cache should have been queried for the injected item"
)
mm_rate = _get_mm_cache_log(llm, caplog_vllm)
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
# Verify cache was queried and injection happened
queries_after = _get_mm_cache_stats(runner.llm.get_metrics())[0]
assert queries_after > queries_before, (
"Cache should have been queried for the injected item"
)
mm_rate = _get_mm_cache_log(runner.llm, caplog_vllm)
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:1]], indirect=True)
def test_inject_into_mm_cache_without_cache(
num_gpus_available,
image_urls,
multimodal_llm_factory,
vllm_runner,
):
"""Test that inject_into_mm_cache works gracefully when processor cache
is disabled (mm_processor_cache_gb=0). Should not crash.
"""
llm = multimodal_llm_factory(
model="llava-hf/llava-1.5-7b-hf",
with vllm_runner(
"llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
mm_processor_cache_gb=0,
)
) as runner:
# Run a normal chat request first to warm up the model.
runner.llm.chat(_make_messages(image_urls[0]))
# Run a normal chat request first to warm up the model.
llm.chat(_make_messages(image_urls[0]))
# Use the renderer to get a proper EngineInput with expanded tokens
renderer = runner.llm.llm_engine.renderer
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[0])],
ChatParams(),
)
eng_input = eng_prompts[0]
# Use the renderer to get a proper EngineInput with expanded tokens
renderer = llm.llm_engine.renderer
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[0])],
ChatParams(),
)
eng_input = eng_prompts[0]
mm_hashes = {"image": ["abcd1234" * 8]}
mm_kwargs = eng_input["mm_kwargs"]
mm_hashes = {"image": ["abcd1234" * 8]}
mm_kwargs = eng_input["mm_kwargs"]
# inject_into_mm_cache should not crash even without cache
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# inject_into_mm_cache should not crash even without cache
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# Build and generate with pre-rendered input
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
# Build and generate with pre-rendered input
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
result = llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
assert len(result) == 1, "Should produce one output"
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
result = runner.llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
assert len(result) == 1, "Should produce one output"
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
tests/entrypoints/multimodal/llm/test_mm_cache_stats.py
+22 -23
View File
@@ -61,38 +61,37 @@ def test_mm_cache_stats(
image_urls,
mm_processor_cache_type,
caplog_vllm,
multimodal_llm_factory,
vllm_runner,
):
llm = multimodal_llm_factory(
model="llava-hf/llava-1.5-7b-hf",
with vllm_runner(
"llava-hf/llava-1.5-7b-hf",
max_model_len=4096,
max_num_seqs=5,
enforce_eager=True,
mm_processor_cache_type=mm_processor_cache_type,
disable_log_stats=False,
limit_mm_per_prompt={"image": 2},
)
) as runner:
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
runner.llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 0)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 0)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (3, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(33.3)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (3, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(33.3)
# NOTE: This only resets hit rate stats in CachingMetrics
# The raw queries and hits counts remain unaffected
runner.llm.reset_mm_cache()
# NOTE: This only resets hit rate stats in CachingMetrics
# The raw queries and hits counts remain unaffected
llm.reset_mm_cache()
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (4, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (4, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(llm.get_metrics()) == (5, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
runner.llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (5, 1)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
tests/entrypoints/multimodal/llm/test_mm_embeds_only.py
+9 -6
View File
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest
from tests.entrypoints.multimodal.conftest import managed_llm
from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
from vllm.exceptions import VLLMValidationError
@@ -14,17 +15,19 @@ TEXT_ONLY_PROMPT = "USER: What is 2 + 2?\nASSISTANT:"
@pytest.fixture(scope="module")
def llm():
def llm(vllm_runner):
"""LLM with enable_mm_embeds=True and all modality limits zeroed out."""
with managed_llm(
model=MODEL,
with vllm_runner(
MODEL,
max_model_len=2048,
enforce_eager=True,
gpu_memory_utilization=0.8,
enable_mm_embeds=True,
limit_mm_per_prompt={"image": 0},
) as llm:
yield llm
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
tests/entrypoints/pooling/basic/test_encode.py
+9 -13
View File
@@ -6,7 +6,6 @@ import weakref
import pytest
from vllm import LLM, PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.exceptions import VLLMValidationError
MODEL_NAME = "intfloat/multilingual-e5-small"
@@ -29,23 +28,20 @@ TOKEN_IDS = [
@pytest.fixture(scope="module")
def llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model=MODEL_NAME,
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
tests/entrypoints/pooling/basic/test_tiling_engine.py
+10 -12
View File
@@ -6,28 +6,26 @@ from unittest import mock
import pytest
from vllm import LLM, PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm import PoolingParams
MODEL_NAME = "intfloat/multilingual-e5-small"
@pytest.fixture(scope="module")
def llm():
llm = LLM(
model=MODEL_NAME,
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_seqs=2, # small to trigger tiling
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
tests/entrypoints/pooling/classify/test_offline.py
+9 -13
View File
@@ -7,7 +7,6 @@ import torch
from tests.models.utils import softmax
from vllm import LLM, ClassificationRequestOutput, PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.tasks import PoolingTask
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
@@ -18,23 +17,20 @@ num_labels = 2
@pytest.fixture(scope="module")
def llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model=MODEL_NAME,
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
tests/entrypoints/pooling/embed/test_offline.py
+10 -13
View File
@@ -7,7 +7,6 @@ import torch
import torch.nn.functional as F
from vllm import LLM, EmbeddingRequestOutput, PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.tasks import PoolingTask
MODEL_NAME = "intfloat/multilingual-e5-small"
@@ -18,23 +17,21 @@ embedding_size = 384
@pytest.fixture(scope="module")
def llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model=MODEL_NAME,
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
)
assert embedding_size == llm.model_config.embedding_size
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
enable_chunked_prefill=None,
) as runner:
assert embedding_size == runner.llm.model_config.embedding_size
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
tests/entrypoints/pooling/reward/test_token_reward_offline.py
+9 -13
View File
@@ -8,7 +8,6 @@ import torch
from tests.models.utils import softmax
from vllm import LLM, PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_NAME = "internlm/internlm2-1_8b-reward"
@@ -16,24 +15,21 @@ prompts = ["The chef prepared a delicious meal."]
@pytest.fixture(scope="module")
def llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model=MODEL_NAME,
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
trust_remote_code=True,
seed=0,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
tests/entrypoints/pooling/scoring/test_bi_encoder_offline.py
+9 -14
View File
@@ -6,8 +6,6 @@ import weakref
import pytest
from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_NAME = "intfloat/multilingual-e5-small"
PROMPT = "The chef prepared a delicious meal."
@@ -27,23 +25,20 @@ DTYPE = "half"
@pytest.fixture(scope="module")
def llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model=MODEL_NAME,
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.fixture(scope="module")
tests/entrypoints/pooling/scoring/test_cross_encoder_offline.py
+9 -13
View File
@@ -8,7 +8,6 @@ import torch
from tests.models.utils import softmax
from vllm import LLM, PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
PROMPT = "The chef prepared a delicious meal."
@@ -24,23 +23,20 @@ TEXTS_2 = [
@pytest.fixture(scope="module")
def llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model=MODEL_NAME,
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.fixture(scope="module")
tests/entrypoints/pooling/scoring/test_late_interaction_offline.py
+9 -15
View File
@@ -5,9 +5,6 @@ import weakref
import pytest
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
from .util import ColBERTScoringHfRunner
MODEL_NAME = "answerdotai/answerai-colbert-small-v1"
@@ -30,23 +27,20 @@ DTYPE = "half"
@pytest.fixture(scope="module")
def llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model=MODEL_NAME,
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.fixture(scope="module")
tests/entrypoints/pooling/scoring/test_late_interaction_offline_vision.py
+9 -15
View File
@@ -5,32 +5,26 @@ import weakref
import pytest
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
from .util import make_base64_image, make_image_mm_param
MODEL_NAME = "vidore/colpali-v1.3-hf"
@pytest.fixture(scope="module")
def llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model=MODEL_NAME,
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
tests/entrypoints/pooling/token_classify/test_offline.py
+9 -13
View File
@@ -6,7 +6,6 @@ import pytest
from vllm import LLM, PoolingRequestOutput
from vllm.config import PoolerConfig
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.tasks import PoolingTask
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
@@ -17,24 +16,21 @@ num_labels = 2
@pytest.fixture(scope="module")
def llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model=MODEL_NAME,
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
pooler_config=PoolerConfig(task="token_classify"),
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
enable_chunked_prefill=None,
) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
tests/entrypoints/pooling/token_embed/test_offline.py
+10 -13
View File
@@ -6,7 +6,6 @@ import pytest
from vllm import LLM, PoolingRequestOutput
from vllm.config import PoolerConfig
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.tasks import PoolingTask
MODEL_NAME = "intfloat/multilingual-e5-small"
@@ -17,24 +16,22 @@ embedding_size = 384
@pytest.fixture(scope="module")
def llm():
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm = LLM(
model=MODEL_NAME,
def llm(vllm_runner):
with vllm_runner(
MODEL_NAME,
max_model_len=None,
pooler_config=PoolerConfig(task="token_embed"),
max_num_batched_tokens=32768,
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
)
assert embedding_size == llm.model_config.embedding_size
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
enable_chunked_prefill=None,
) as runner:
assert embedding_size == runner.llm.model_config.embedding_size
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup
tests/entrypoints/weight_transfer/test_weight_transfer_llm.py
+47 -40
View File
@@ -8,13 +8,13 @@ actual NCCL communication.
"""
import os
import weakref
from dataclasses import dataclass
from unittest.mock import patch
import pytest
import torch
from vllm import LLM
from vllm.config import WeightTransferConfig
from vllm.distributed.weight_transfer.base import (
WeightTransferEngine,
@@ -102,25 +102,24 @@ def mock_create_engine(config, vllm_config, device, model):
@create_new_process_for_each_test()
def test_get_world_size_tp1():
def test_get_world_size_tp1(vllm_runner):
"""Test world_size is correctly configured for TP=1."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
llm = LLM(
model=MODEL_NAME,
with vllm_runner(
MODEL_NAME,
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
)
world_size = llm.llm_engine.vllm_config.parallel_config.world_size
assert world_size == 1
) as runner:
world_size = runner.llm.llm_engine.vllm_config.parallel_config.world_size
assert world_size == 1
@create_new_process_for_each_test()
def test_init_weight_transfer_engine_calls_engine():
def test_init_weight_transfer_engine_calls_engine(vllm_runner):
"""Test that init_weight_transfer_engine calls the engine's
init_transfer_engine method."""
if torch.accelerator.device_count() < 1:
@@ -131,17 +130,20 @@ def test_init_weight_transfer_engine_calls_engine():
# Enable insecure serialization to allow pickling functions for collective_rpc
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
with patch(
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
mock_create_engine,
):
llm = LLM(
model=MODEL_NAME,
with (
patch(
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
mock_create_engine,
),
vllm_runner(
MODEL_NAME,
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
)
) as runner,
):
llm = weakref.proxy(runner.llm)
# Verify engine was created
def check_engine_exists(self):
@@ -170,7 +172,7 @@ def test_init_weight_transfer_engine_calls_engine():
@create_new_process_for_each_test()
def test_update_weights_calls_engine():
def test_update_weights_calls_engine(vllm_runner):
"""Test that update_weights calls the engine's receive_weights method."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -180,17 +182,20 @@ def test_update_weights_calls_engine():
# Enable insecure serialization to allow pickling functions for collective_rpc
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
with patch(
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
mock_create_engine,
):
llm = LLM(
model=MODEL_NAME,
with (
patch(
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
mock_create_engine,
),
vllm_runner(
MODEL_NAME,
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
)
) as runner,
):
llm = weakref.proxy(runner.llm)
# First init the weight transfer
llm.init_weight_transfer_engine(
@@ -233,7 +238,7 @@ def test_update_weights_calls_engine():
@create_new_process_for_each_test()
def test_full_weight_transfer_flow():
def test_full_weight_transfer_flow(vllm_runner):
"""Test the complete weight transfer flow: init -> start -> update -> finish."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -243,17 +248,20 @@ def test_full_weight_transfer_flow():
# Enable insecure serialization to allow pickling functions for collective_rpc
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
with patch(
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
mock_create_engine,
):
llm = LLM(
model=MODEL_NAME,
with (
patch(
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
mock_create_engine,
),
vllm_runner(
MODEL_NAME,
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
)
) as runner,
):
llm = weakref.proxy(runner.llm)
assert llm.get_weight_version() == "default"
@@ -309,20 +317,19 @@ def test_full_weight_transfer_flow():
@create_new_process_for_each_test()
def test_weight_transfer_config_backend():
def test_weight_transfer_config_backend(vllm_runner):
"""Test that WeightTransferConfig backend is properly configured."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
# Test with nccl backend
llm = LLM(
model=MODEL_NAME,
with vllm_runner(
MODEL_NAME,
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
)
config = llm.llm_engine.vllm_config.weight_transfer_config
assert config is not None
assert config.backend == "nccl"
) as runner:
config = runner.llm.llm_engine.vllm_config.weight_transfer_config
assert config is not None
assert config.backend == "nccl"