[ROCm][CI] Solidify entrypoint LLM lifecycle (#51280)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-08-13 12:24:02 -05:00
committed by GitHub
parent 69d4c3a06b
commit b96bcd0b47
24 changed files with 1002 additions and 1103 deletions
@@ -9,9 +9,6 @@ import pytest
import regex as re import regex as re
import urllib3 import urllib3
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_CONFIGS = [ MODEL_CONFIGS = [
{ {
"model": "facebook/opt-125m", "model": "facebook/opt-125m",
@@ -55,19 +52,26 @@ MODEL_CONFIGS = [
] ]
def _create_runner(vllm_runner, model_config):
runner_config = model_config.copy()
model = runner_config.pop("model")
tokenizer_name = runner_config.pop("tokenizer", None)
return vllm_runner(model, tokenizer_name=tokenizer_name, **runner_config)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def cache_models(): def cache_models(vllm_runner):
# Cache model files first # Cache model files first
for model_config in MODEL_CONFIGS: for model_config in MODEL_CONFIGS:
LLM(**model_config) with _create_runner(vllm_runner, model_config):
cleanup_dist_env_and_memory() pass
yield yield
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@pytest.mark.usefixtures("cache_models") @pytest.mark.usefixtures("cache_models")
def test_offline_mode(monkeypatch: pytest.MonkeyPatch): def test_offline_mode(monkeypatch: pytest.MonkeyPatch, vllm_runner):
# Set HF to offline mode and ensure we can still construct an LLM # Set HF to offline mode and ensure we can still construct an LLM
with monkeypatch.context() as m: with monkeypatch.context() as m:
try: try:
@@ -93,7 +97,8 @@ def test_offline_mode(monkeypatch: pytest.MonkeyPatch):
_re_import_modules() _re_import_modules()
# Cached model files should be used in offline mode # Cached model files should be used in offline mode
for model_config in MODEL_CONFIGS: for model_config in MODEL_CONFIGS:
LLM(**model_config) with _create_runner(vllm_runner, model_config):
pass
finally: finally:
# Reset the environment after the test # Reset the environment after the test
# NB: Assuming tests are run in online mode # NB: Assuming tests are run in online mode
@@ -136,7 +141,7 @@ def _re_import_modules():
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@pytest.mark.usefixtures("cache_models") @pytest.mark.usefixtures("cache_models")
def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch): def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch, vllm_runner):
# Set HF to offline mode and ensure we can still construct an LLM # Set HF to offline mode and ensure we can still construct an LLM
with monkeypatch.context() as m: with monkeypatch.context() as m:
try: try:
@@ -159,7 +164,8 @@ def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch):
# Need to re-import huggingface_hub # Need to re-import huggingface_hub
# and friends to set up offline mode # and friends to set up offline mode
_re_import_modules() _re_import_modules()
LLM(model="facebook/opt-125m") with vllm_runner("facebook/opt-125m"):
pass
finally: finally:
# Reset the environment after the test # Reset the environment after the test
# NB: Assuming tests are run in online mode # NB: Assuming tests are run in online mode
+21 -36
View File
@@ -4,46 +4,36 @@ import weakref
import pytest import pytest
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.exceptions import VLLMValidationError from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import SamplingParams from vllm.sampling_params import SamplingParams
@pytest.fixture(scope="function") @pytest.fixture(scope="function")
def text_llm(): def text_llm(vllm_runner):
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection "meta-llama/Llama-3.2-1B-Instruct", enforce_eager=True, seed=0
llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct", enforce_eager=True, seed=0) ) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
yield weakref.proxy(llm) # avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
del llm
cleanup_dist_env_and_memory()
@pytest.fixture(scope="function") @pytest.fixture(scope="function")
def llm_for_failure_test(): def llm_for_failure_test(vllm_runner):
""" """
Fixture for testing issue #26081. Fixture for testing issue #26081.
Uses a small max_model_len to easily trigger length errors. Uses a small max_model_len to easily trigger length errors.
""" """
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection "meta-llama/Llama-3.2-1B-Instruct",
llm = LLM(
model="meta-llama/Llama-3.2-1B-Instruct",
enforce_eager=True, enforce_eager=True,
seed=0, seed=0,
max_model_len=128, max_model_len=128,
disable_log_stats=True, disable_log_stats=True,
) ) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
yield weakref.proxy(llm) # avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
del llm
cleanup_dist_env_and_memory()
def test_chat(text_llm): def test_chat(text_llm):
@@ -99,21 +89,16 @@ def test_llm_chat_tokenization_no_double_bos(text_llm):
@pytest.fixture(scope="function") @pytest.fixture(scope="function")
def thinking_llm(): def thinking_llm(vllm_runner):
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection "Qwen/Qwen3-0.6B",
llm = LLM(
model="Qwen/Qwen3-0.6B",
max_model_len=4096, max_model_len=4096,
enforce_eager=True, enforce_eager=True,
seed=0, seed=0,
) ) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
yield weakref.proxy(llm) # avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
del llm
cleanup_dist_env_and_memory()
@pytest.mark.parametrize("enable_thinking", [True, False]) @pytest.mark.parametrize("enable_thinking", [True, False])
+5 -7
View File
@@ -4,15 +4,13 @@
import pytest import pytest
import torch import torch
from vllm import LLM
from ...utils import create_new_process_for_each_test from ...utils import create_new_process_for_each_test
@pytest.mark.parametrize("tp_size", [1, 2]) @pytest.mark.parametrize("tp_size", [1, 2])
@pytest.mark.parametrize("backend", ["mp", "ray"]) @pytest.mark.parametrize("backend", ["mp", "ray"])
@create_new_process_for_each_test() @create_new_process_for_each_test()
def test_collective_rpc(tp_size, backend, monkeypatch): def test_collective_rpc(tp_size, backend, monkeypatch, vllm_runner):
if torch.accelerator.device_count() < tp_size: if torch.accelerator.device_count() < tp_size:
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}") pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
if tp_size == 1 and backend == "ray": if tp_size == 1 and backend == "ray":
@@ -26,11 +24,11 @@ def test_collective_rpc(tp_size, backend, monkeypatch):
return self.rank return self.rank
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
llm = LLM( with vllm_runner(
model="hmellor/tiny-random-LlamaForCausalLM", "hmellor/tiny-random-LlamaForCausalLM",
enforce_eager=True, enforce_eager=True,
load_format="dummy", load_format="dummy",
tensor_parallel_size=tp_size, tensor_parallel_size=tp_size,
distributed_executor_backend=backend, distributed_executor_backend=backend,
) ) as runner:
assert llm.collective_rpc(echo_rank) == list(range(tp_size)) assert runner.llm.collective_rpc(echo_rank) == list(range(tp_size))
+28 -34
View File
@@ -6,7 +6,6 @@ import weakref
import pytest import pytest
from vllm import LLM, SamplingParams from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_NAME = "distilbert/distilgpt2" MODEL_NAME = "distilbert/distilgpt2"
@@ -26,22 +25,17 @@ TOKEN_IDS = [
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def llm(): def llm(vllm_runner):
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection MODEL_NAME,
llm = LLM(
model=MODEL_NAME,
max_num_batched_tokens=4096, max_num_batched_tokens=4096,
tensor_parallel_size=1, tensor_parallel_size=1,
gpu_memory_utilization=0.10, gpu_memory_utilization=0.10,
enforce_eager=True, enforce_eager=True,
) ) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
yield weakref.proxy(llm) # avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
del llm
cleanup_dist_env_and_memory()
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@@ -97,34 +91,34 @@ def test_single_prompt_priority(llm: LLM):
assert len(outputs) == 1 assert len(outputs) == 1
def test_max_model_len(): def test_max_model_len(vllm_runner):
max_model_len = 20 max_model_len = 20
llm = LLM( with vllm_runner(
model=MODEL_NAME, MODEL_NAME,
max_model_len=max_model_len, max_model_len=max_model_len,
gpu_memory_utilization=0.10, gpu_memory_utilization=0.10,
enforce_eager=True, # reduce test time enforce_eager=True, # reduce test time
) ) as runner:
sampling_params = SamplingParams(max_tokens=max_model_len + 10) sampling_params = SamplingParams(max_tokens=max_model_len + 10)
outputs = llm.generate(PROMPTS, sampling_params) outputs = runner.llm.generate(PROMPTS, sampling_params)
for output in outputs: for output in outputs:
num_total_tokens = len(output.prompt_token_ids) + len( num_total_tokens = len(output.prompt_token_ids) + len(
output.outputs[0].token_ids output.outputs[0].token_ids
) )
# Total tokens must not exceed max_model_len. # Total tokens must not exceed max_model_len.
# It can be less if generation finishes due to other reasons (e.g., EOS) # It can be less if generation finishes due to other reasons (e.g., EOS)
# before reaching the absolute model length limit. # before reaching the absolute model length limit.
assert num_total_tokens <= max_model_len assert num_total_tokens <= max_model_len
def test_log_stats(): def test_log_stats(vllm_runner):
llm = LLM( with vllm_runner(
model=MODEL_NAME, MODEL_NAME,
disable_log_stats=False, disable_log_stats=False,
gpu_memory_utilization=0.10, gpu_memory_utilization=0.10,
enforce_eager=True, # reduce test time enforce_eager=True, # reduce test time
) ) as runner:
outputs = llm.generate(PROMPTS, sampling_params=None) outputs = runner.llm.generate(PROMPTS, sampling_params=None)
# disable_log_stats is False, every output should have metrics # disable_log_stats is False, every output should have metrics
assert all(output.metrics is not None for output in outputs) assert all(output.metrics is not None for output in outputs)
+25 -12
View File
@@ -1,10 +1,10 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm import LLM, SamplingParams from vllm import SamplingParams
def test_gpu_memory_utilization(): def test_gpu_memory_utilization(vllm_runner):
prompts = [ prompts = [
"Hello, my name is", "Hello, my name is",
"The president of the United States is", "The president of the United States is",
@@ -15,13 +15,26 @@ def test_gpu_memory_utilization():
# makes sure gpu_memory_utilization is per-instance limit, # makes sure gpu_memory_utilization is per-instance limit,
# not a global limit # not a global limit
llms = [ with (
LLM(model="facebook/opt-125m", gpu_memory_utilization=0.3, enforce_eager=True) vllm_runner(
for i in range(3) "facebook/opt-125m",
] gpu_memory_utilization=0.3,
for llm in llms: enforce_eager=True,
outputs = llm.generate(prompts, sampling_params) ) as runner_0,
for output in outputs: vllm_runner(
prompt = output.prompt "facebook/opt-125m",
generated_text = output.outputs[0].text gpu_memory_utilization=0.3,
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") enforce_eager=True,
) as runner_1,
vllm_runner(
"facebook/opt-125m",
gpu_memory_utilization=0.3,
enforce_eager=True,
) as runner_2,
):
for runner in (runner_0, runner_1, runner_2):
outputs = runner.llm.generate(prompts, sampling_params)
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+22 -17
View File
@@ -4,30 +4,35 @@
import pytest import pytest
import torch import torch
from vllm import LLM
from vllm.exceptions import VLLMValidationError from vllm.exceptions import VLLMValidationError
def test_empty_prompt(): def test_empty_prompt(vllm_runner):
llm = LLM(model="openai-community/gpt2", enforce_eager=True) with (
with pytest.raises(VLLMValidationError, match="decoder prompt cannot be empty"): vllm_runner("openai-community/gpt2", enforce_eager=True) as runner,
llm.generate([""]) pytest.raises(VLLMValidationError, match="decoder prompt cannot be empty"),
):
runner.llm.generate([""])
def test_out_of_vocab_token(): def test_out_of_vocab_token(vllm_runner):
llm = LLM(model="openai-community/gpt2", enforce_eager=True) with (
with pytest.raises(VLLMValidationError, match="out of vocabulary"): vllm_runner("openai-community/gpt2", enforce_eager=True) as runner,
llm.generate({"prompt_token_ids": [999999]}) pytest.raises(VLLMValidationError, match="out of vocabulary"),
):
runner.llm.generate({"prompt_token_ids": [999999]})
def test_require_mm_embeds(): def test_require_mm_embeds(vllm_runner):
llm = LLM( with (
model="llava-hf/llava-1.5-7b-hf", vllm_runner(
enforce_eager=True, "llava-hf/llava-1.5-7b-hf",
enable_mm_embeds=False, enforce_eager=True,
) enable_mm_embeds=False,
with pytest.raises(ValueError, match="--enable-mm-embeds"): ) as runner,
llm.generate( pytest.raises(ValueError, match="--enable-mm-embeds"),
):
runner.llm.generate(
{ {
"prompt": "<image>", "prompt": "<image>",
"multi_modal_data": {"image": torch.empty(1, 1, 1)}, "multi_modal_data": {"image": torch.empty(1, 1, 1)},
File diff suppressed because it is too large Load Diff
-73
View File
@@ -1,11 +1,5 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from typing import Any
import pytest
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA) # Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
TEST_IMAGE_ASSETS = [ TEST_IMAGE_ASSETS = [
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" "2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
@@ -13,70 +7,3 @@ TEST_IMAGE_ASSETS = [
"1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png", "1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
"RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png", "RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
] ]
def _shutdown_llm(llm: Any, gpu_memory_utilization: float) -> None:
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.platforms import current_platform
try:
shutdown_timeout = 60.0 if current_platform.is_rocm() else None
llm.llm_engine.engine_core.shutdown(timeout=shutdown_timeout)
except Exception:
pass
del llm
try:
import torch
torch._dynamo.reset()
except Exception:
pass
cleanup_dist_env_and_memory()
if current_platform.is_rocm():
from tests.utils import wait_for_rocm_memory_to_settle
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
@contextmanager
def managed_llm(*args: Any, **kwargs: Any) -> Iterator[Any]:
from vllm import LLM
llm = LLM(*args, **kwargs)
gpu_memory_utilization = (
llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
)
try:
yield llm
finally:
_shutdown_llm(llm, gpu_memory_utilization)
def _make_managed_llm_factory() -> Iterator[Callable[..., Any]]:
from vllm import LLM
llms: list[tuple[Any, float]] = []
def make_llm(*args: Any, **kwargs: Any) -> Any:
llm = LLM(*args, **kwargs)
gpu_memory_utilization = (
llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
)
llms.append((llm, gpu_memory_utilization))
return llm
try:
yield make_llm
finally:
while llms:
llm, gpu_memory_utilization = llms.pop()
_shutdown_llm(llm, gpu_memory_utilization)
@pytest.fixture
def multimodal_llm_factory() -> Iterator[Callable[..., Any]]:
yield from _make_managed_llm_factory()
@@ -1,21 +1,26 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest import pytest
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
@pytest.fixture(scope="function") @pytest.fixture(scope="function")
def vision_llm(multimodal_llm_factory): def vision_llm(vllm_runner):
return multimodal_llm_factory( with vllm_runner(
model="microsoft/Phi-3.5-vision-instruct", "microsoft/Phi-3.5-vision-instruct",
max_model_len=4096, max_model_len=4096,
max_num_seqs=5, max_num_seqs=5,
enforce_eager=True, enforce_eager=True,
trust_remote_code=True, trust_remote_code=True,
limit_mm_per_prompt={"image": 2}, limit_mm_per_prompt={"image": 2},
seed=0, seed=0,
) ) as runner:
# pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.parametrize( @pytest.mark.parametrize(
@@ -69,7 +69,7 @@ def test_inject_into_mm_cache(
image_urls, image_urls,
mm_processor_cache_type, mm_processor_cache_type,
caplog_vllm, caplog_vllm,
multimodal_llm_factory, vllm_runner,
): ):
"""Test that inject_into_mm_cache() injects pre-processed mm_kwargs into """Test that inject_into_mm_cache() injects pre-processed mm_kwargs into
the processor cache and MM cache hit metrics are updated correctly. the processor cache and MM cache hit metrics are updated correctly.
@@ -79,117 +79,115 @@ def test_inject_into_mm_cache(
2. Extract cached kwargs, call inject_into_mm_cache with a new hash, 2. Extract cached kwargs, call inject_into_mm_cache with a new hash,
then generate with a pre-rendered input -> verifies injection works then generate with a pre-rendered input -> verifies injection works
""" """
llm = multimodal_llm_factory( with vllm_runner(
model="llava-hf/llava-1.5-7b-hf", "llava-hf/llava-1.5-7b-hf",
max_model_len=4096, max_model_len=4096,
max_num_seqs=5, max_num_seqs=5,
enforce_eager=True, enforce_eager=True,
disable_log_stats=False, disable_log_stats=False,
limit_mm_per_prompt={"image": 2}, limit_mm_per_prompt={"image": 2},
mm_processor_cache_type=mm_processor_cache_type, mm_processor_cache_type=mm_processor_cache_type,
) ) as runner:
# Step 1: Normal requests to populate the cache
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
# Step 1: Normal requests to populate the cache runner.llm.chat(_make_messages(image_urls[0]))
llm.chat(_make_messages(image_urls[0])) assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 1)
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0) assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(50.0)
llm.chat(_make_messages(image_urls[0])) # Step 2: Use a second image to get valid expanded tokens and
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 1) # placeholder positions via the renderer.
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(50.0) runner.llm.chat(_make_messages(image_urls[1]))
queries_before = _get_mm_cache_stats(runner.llm.get_metrics())[0] # 3
# Step 2: Use a second image to get valid expanded tokens and renderer = runner.llm.llm_engine.renderer
# placeholder positions via the renderer. cache = renderer.mm_processor_cache
llm.chat(_make_messages(image_urls[1])) assert cache is not None, "Processor cache should be enabled"
queries_before = _get_mm_cache_stats(llm.get_metrics())[0] # 3
renderer = llm.llm_engine.renderer _, eng_prompts = renderer.render_chat(
cache = renderer.mm_processor_cache [_make_messages(image_urls[1])],
assert cache is not None, "Processor cache should be enabled" ChatParams(),
)
eng_input = eng_prompts[0]
_, eng_prompts = renderer.render_chat( # Inject pre-processed mm_kwargs with a NEW hash via public API
[_make_messages(image_urls[1])], new_mm_hash = "deadbeef" * 8
ChatParams(), mm_hashes = {"image": [new_mm_hash]}
) mm_kwargs = eng_input["mm_kwargs"]
eng_input = eng_prompts[0]
# Inject pre-processed mm_kwargs with a NEW hash via public API runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
new_mm_hash = "deadbeef" * 8
mm_hashes = {"image": [new_mm_hash]}
mm_kwargs = eng_input["mm_kwargs"]
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs) # Build pre-rendered input (no externally_processed flag needed)
pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
# Build pre-rendered input (no externally_processed flag needed) runner.llm.generate(
pre_rendered_input = { pre_rendered_input,
"type": "multimodal", sampling_params=SamplingParams(max_tokens=1),
"prompt_token_ids": eng_input["prompt_token_ids"], )
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
llm.generate( # Verify cache was queried and injection happened
pre_rendered_input, queries_after = _get_mm_cache_stats(runner.llm.get_metrics())[0]
sampling_params=SamplingParams(max_tokens=1), assert queries_after > queries_before, (
) "Cache should have been queried for the injected item"
)
# Verify cache was queried and injection happened mm_rate = _get_mm_cache_log(runner.llm, caplog_vllm)
queries_after = _get_mm_cache_stats(llm.get_metrics())[0] assert mm_rate >= 0.0, "MM cache hit rate should be reported"
assert queries_after > queries_before, (
"Cache should have been queried for the injected item"
)
mm_rate = _get_mm_cache_log(llm, caplog_vllm)
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:1]], indirect=True) @pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:1]], indirect=True)
def test_inject_into_mm_cache_without_cache( def test_inject_into_mm_cache_without_cache(
num_gpus_available, num_gpus_available,
image_urls, image_urls,
multimodal_llm_factory, vllm_runner,
): ):
"""Test that inject_into_mm_cache works gracefully when processor cache """Test that inject_into_mm_cache works gracefully when processor cache
is disabled (mm_processor_cache_gb=0). Should not crash. is disabled (mm_processor_cache_gb=0). Should not crash.
""" """
llm = multimodal_llm_factory( with vllm_runner(
model="llava-hf/llava-1.5-7b-hf", "llava-hf/llava-1.5-7b-hf",
max_model_len=4096, max_model_len=4096,
max_num_seqs=5, max_num_seqs=5,
enforce_eager=True, enforce_eager=True,
disable_log_stats=False, disable_log_stats=False,
limit_mm_per_prompt={"image": 2}, limit_mm_per_prompt={"image": 2},
mm_processor_cache_gb=0, mm_processor_cache_gb=0,
) ) as runner:
# Run a normal chat request first to warm up the model.
runner.llm.chat(_make_messages(image_urls[0]))
# Run a normal chat request first to warm up the model. # Use the renderer to get a proper EngineInput with expanded tokens
llm.chat(_make_messages(image_urls[0])) renderer = runner.llm.llm_engine.renderer
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[0])],
ChatParams(),
)
eng_input = eng_prompts[0]
# Use the renderer to get a proper EngineInput with expanded tokens mm_hashes = {"image": ["abcd1234" * 8]}
renderer = llm.llm_engine.renderer mm_kwargs = eng_input["mm_kwargs"]
_, eng_prompts = renderer.render_chat(
[_make_messages(image_urls[0])],
ChatParams(),
)
eng_input = eng_prompts[0]
mm_hashes = {"image": ["abcd1234" * 8]} # inject_into_mm_cache should not crash even without cache
mm_kwargs = eng_input["mm_kwargs"] runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
# inject_into_mm_cache should not crash even without cache # Build and generate with pre-rendered input
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs) pre_rendered_input = {
"type": "multimodal",
"prompt_token_ids": eng_input["prompt_token_ids"],
"mm_kwargs": mm_kwargs,
"mm_hashes": mm_hashes,
"mm_placeholders": eng_input["mm_placeholders"],
}
# Build and generate with pre-rendered input result = runner.llm.generate(
pre_rendered_input = { pre_rendered_input,
"type": "multimodal", sampling_params=SamplingParams(max_tokens=1),
"prompt_token_ids": eng_input["prompt_token_ids"], )
"mm_kwargs": mm_kwargs, assert len(result) == 1, "Should produce one output"
"mm_hashes": mm_hashes, assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
"mm_placeholders": eng_input["mm_placeholders"],
}
result = llm.generate(
pre_rendered_input,
sampling_params=SamplingParams(max_tokens=1),
)
assert len(result) == 1, "Should produce one output"
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
@@ -61,38 +61,37 @@ def test_mm_cache_stats(
image_urls, image_urls,
mm_processor_cache_type, mm_processor_cache_type,
caplog_vllm, caplog_vllm,
multimodal_llm_factory, vllm_runner,
): ):
llm = multimodal_llm_factory( with vllm_runner(
model="llava-hf/llava-1.5-7b-hf", "llava-hf/llava-1.5-7b-hf",
max_model_len=4096, max_model_len=4096,
max_num_seqs=5, max_num_seqs=5,
enforce_eager=True, enforce_eager=True,
mm_processor_cache_type=mm_processor_cache_type, mm_processor_cache_type=mm_processor_cache_type,
disable_log_stats=False, disable_log_stats=False,
limit_mm_per_prompt={"image": 2}, limit_mm_per_prompt={"image": 2},
) ) as runner:
runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[0])) runner.llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0) assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 0)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0) assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[1])) runner.llm.chat(_make_messages(image_urls[0]))
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 0) assert _get_mm_cache_stats(runner.llm.get_metrics()) == (3, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0) assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(33.3)
llm.chat(_make_messages(image_urls[0])) # NOTE: This only resets hit rate stats in CachingMetrics
assert _get_mm_cache_stats(llm.get_metrics()) == (3, 1) # The raw queries and hits counts remain unaffected
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(33.3) runner.llm.reset_mm_cache()
# NOTE: This only resets hit rate stats in CachingMetrics runner.llm.chat(_make_messages(image_urls[0]))
# The raw queries and hits counts remain unaffected assert _get_mm_cache_stats(runner.llm.get_metrics()) == (4, 1)
llm.reset_mm_cache() assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[0])) runner.llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(llm.get_metrics()) == (4, 1) assert _get_mm_cache_stats(runner.llm.get_metrics()) == (5, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0) assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
llm.chat(_make_messages(image_urls[1]))
assert _get_mm_cache_stats(llm.get_metrics()) == (5, 1)
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
import pytest import pytest
from tests.entrypoints.multimodal.conftest import managed_llm
from vllm import LLM, SamplingParams from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset from vllm.assets.image import ImageAsset
from vllm.exceptions import VLLMValidationError from vllm.exceptions import VLLMValidationError
@@ -14,17 +15,19 @@ TEXT_ONLY_PROMPT = "USER: What is 2 + 2?\nASSISTANT:"
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def llm(): def llm(vllm_runner):
"""LLM with enable_mm_embeds=True and all modality limits zeroed out.""" """LLM with enable_mm_embeds=True and all modality limits zeroed out."""
with managed_llm( with vllm_runner(
model=MODEL, MODEL,
max_model_len=2048, max_model_len=2048,
enforce_eager=True, enforce_eager=True,
gpu_memory_utilization=0.8, gpu_memory_utilization=0.8,
enable_mm_embeds=True, enable_mm_embeds=True,
limit_mm_per_prompt={"image": 0}, limit_mm_per_prompt={"image": 0},
) as llm: ) as runner:
yield llm # pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
yield weakref.proxy(runner.llm)
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
+9 -13
View File
@@ -6,7 +6,6 @@ import weakref
import pytest import pytest
from vllm import LLM, PoolingParams from vllm import LLM, PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.exceptions import VLLMValidationError from vllm.exceptions import VLLMValidationError
MODEL_NAME = "intfloat/multilingual-e5-small" MODEL_NAME = "intfloat/multilingual-e5-small"
@@ -29,23 +28,20 @@ TOKEN_IDS = [
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def llm(): def llm(vllm_runner):
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection MODEL_NAME,
llm = LLM( max_model_len=None,
model=MODEL_NAME,
max_num_batched_tokens=32768, max_num_batched_tokens=32768,
tensor_parallel_size=1, tensor_parallel_size=1,
gpu_memory_utilization=0.75, gpu_memory_utilization=0.75,
enforce_eager=True, enforce_eager=True,
seed=0, seed=0,
) enable_chunked_prefill=None,
) as runner:
yield weakref.proxy(llm) # pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
del llm yield weakref.proxy(runner.llm)
cleanup_dist_env_and_memory()
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@@ -6,28 +6,26 @@ from unittest import mock
import pytest import pytest
from vllm import LLM, PoolingParams from vllm import PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_NAME = "intfloat/multilingual-e5-small" MODEL_NAME = "intfloat/multilingual-e5-small"
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def llm(): def llm(vllm_runner):
llm = LLM( with vllm_runner(
model=MODEL_NAME, MODEL_NAME,
max_model_len=None,
max_num_seqs=2, # small to trigger tiling max_num_seqs=2, # small to trigger tiling
tensor_parallel_size=1, tensor_parallel_size=1,
gpu_memory_utilization=0.75, gpu_memory_utilization=0.75,
enforce_eager=True, enforce_eager=True,
seed=0, seed=0,
) enable_chunked_prefill=None,
) as runner:
yield weakref.proxy(llm) # pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
del llm yield weakref.proxy(runner.llm)
cleanup_dist_env_and_memory()
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@@ -7,7 +7,6 @@ import torch
from tests.models.utils import softmax from tests.models.utils import softmax
from vllm import LLM, ClassificationRequestOutput, PoolingParams from vllm import LLM, ClassificationRequestOutput, PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.tasks import PoolingTask from vllm.tasks import PoolingTask
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach" MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
@@ -18,23 +17,20 @@ num_labels = 2
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def llm(): def llm(vllm_runner):
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection MODEL_NAME,
llm = LLM( max_model_len=None,
model=MODEL_NAME,
max_num_batched_tokens=32768, max_num_batched_tokens=32768,
tensor_parallel_size=1, tensor_parallel_size=1,
gpu_memory_utilization=0.75, gpu_memory_utilization=0.75,
enforce_eager=True, enforce_eager=True,
seed=0, seed=0,
) enable_chunked_prefill=None,
) as runner:
yield weakref.proxy(llm) # pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
del llm yield weakref.proxy(runner.llm)
cleanup_dist_env_and_memory()
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
+10 -13
View File
@@ -7,7 +7,6 @@ import torch
import torch.nn.functional as F import torch.nn.functional as F
from vllm import LLM, EmbeddingRequestOutput, PoolingParams from vllm import LLM, EmbeddingRequestOutput, PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.tasks import PoolingTask from vllm.tasks import PoolingTask
MODEL_NAME = "intfloat/multilingual-e5-small" MODEL_NAME = "intfloat/multilingual-e5-small"
@@ -18,23 +17,21 @@ embedding_size = 384
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def llm(): def llm(vllm_runner):
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection MODEL_NAME,
llm = LLM( max_model_len=None,
model=MODEL_NAME,
max_num_batched_tokens=32768, max_num_batched_tokens=32768,
tensor_parallel_size=1, tensor_parallel_size=1,
gpu_memory_utilization=0.75, gpu_memory_utilization=0.75,
enforce_eager=True, enforce_eager=True,
seed=0, seed=0,
) enable_chunked_prefill=None,
assert embedding_size == llm.model_config.embedding_size ) as runner:
assert embedding_size == runner.llm.model_config.embedding_size
yield weakref.proxy(llm) # pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
del llm yield weakref.proxy(runner.llm)
cleanup_dist_env_and_memory()
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@@ -8,7 +8,6 @@ import torch
from tests.models.utils import softmax from tests.models.utils import softmax
from vllm import LLM, PoolingParams from vllm import LLM, PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_NAME = "internlm/internlm2-1_8b-reward" MODEL_NAME = "internlm/internlm2-1_8b-reward"
@@ -16,24 +15,21 @@ prompts = ["The chef prepared a delicious meal."]
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def llm(): def llm(vllm_runner):
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection MODEL_NAME,
llm = LLM( max_model_len=None,
model=MODEL_NAME,
max_num_batched_tokens=32768, max_num_batched_tokens=32768,
tensor_parallel_size=1, tensor_parallel_size=1,
gpu_memory_utilization=0.75, gpu_memory_utilization=0.75,
enforce_eager=True, enforce_eager=True,
trust_remote_code=True, trust_remote_code=True,
seed=0, seed=0,
) enable_chunked_prefill=None,
) as runner:
yield weakref.proxy(llm) # pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
del llm yield weakref.proxy(runner.llm)
cleanup_dist_env_and_memory()
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@@ -6,8 +6,6 @@ import weakref
import pytest import pytest
from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_NAME = "intfloat/multilingual-e5-small" MODEL_NAME = "intfloat/multilingual-e5-small"
PROMPT = "The chef prepared a delicious meal." PROMPT = "The chef prepared a delicious meal."
@@ -27,23 +25,20 @@ DTYPE = "half"
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def llm(): def llm(vllm_runner):
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection MODEL_NAME,
llm = LLM( max_model_len=None,
model=MODEL_NAME,
max_num_batched_tokens=32768, max_num_batched_tokens=32768,
tensor_parallel_size=1, tensor_parallel_size=1,
gpu_memory_utilization=0.75, gpu_memory_utilization=0.75,
enforce_eager=True, enforce_eager=True,
seed=0, seed=0,
) enable_chunked_prefill=None,
) as runner:
yield weakref.proxy(llm) # pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
del llm yield weakref.proxy(runner.llm)
cleanup_dist_env_and_memory()
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@@ -8,7 +8,6 @@ import torch
from tests.models.utils import softmax from tests.models.utils import softmax
from vllm import LLM, PoolingParams from vllm import LLM, PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls" MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
PROMPT = "The chef prepared a delicious meal." PROMPT = "The chef prepared a delicious meal."
@@ -24,23 +23,20 @@ TEXTS_2 = [
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def llm(): def llm(vllm_runner):
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection MODEL_NAME,
llm = LLM( max_model_len=None,
model=MODEL_NAME,
max_num_batched_tokens=32768, max_num_batched_tokens=32768,
tensor_parallel_size=1, tensor_parallel_size=1,
gpu_memory_utilization=0.75, gpu_memory_utilization=0.75,
enforce_eager=True, enforce_eager=True,
seed=0, seed=0,
) enable_chunked_prefill=None,
) as runner:
yield weakref.proxy(llm) # pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
del llm yield weakref.proxy(runner.llm)
cleanup_dist_env_and_memory()
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@@ -5,9 +5,6 @@ import weakref
import pytest import pytest
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
from .util import ColBERTScoringHfRunner from .util import ColBERTScoringHfRunner
MODEL_NAME = "answerdotai/answerai-colbert-small-v1" MODEL_NAME = "answerdotai/answerai-colbert-small-v1"
@@ -30,23 +27,20 @@ DTYPE = "half"
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def llm(): def llm(vllm_runner):
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection MODEL_NAME,
llm = LLM( max_model_len=None,
model=MODEL_NAME,
max_num_batched_tokens=32768, max_num_batched_tokens=32768,
tensor_parallel_size=1, tensor_parallel_size=1,
gpu_memory_utilization=0.75, gpu_memory_utilization=0.75,
enforce_eager=True, enforce_eager=True,
seed=0, seed=0,
) enable_chunked_prefill=None,
) as runner:
yield weakref.proxy(llm) # pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
del llm yield weakref.proxy(runner.llm)
cleanup_dist_env_and_memory()
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@@ -5,32 +5,26 @@ import weakref
import pytest import pytest
from vllm import LLM
from vllm.distributed import cleanup_dist_env_and_memory
from .util import make_base64_image, make_image_mm_param from .util import make_base64_image, make_image_mm_param
MODEL_NAME = "vidore/colpali-v1.3-hf" MODEL_NAME = "vidore/colpali-v1.3-hf"
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def llm(): def llm(vllm_runner):
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection MODEL_NAME,
llm = LLM( max_model_len=None,
model=MODEL_NAME,
max_num_batched_tokens=32768, max_num_batched_tokens=32768,
tensor_parallel_size=1, tensor_parallel_size=1,
gpu_memory_utilization=0.75, gpu_memory_utilization=0.75,
enforce_eager=True, enforce_eager=True,
seed=0, seed=0,
) enable_chunked_prefill=None,
) as runner:
yield weakref.proxy(llm) # pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
del llm yield weakref.proxy(runner.llm)
cleanup_dist_env_and_memory()
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@@ -6,7 +6,6 @@ import pytest
from vllm import LLM, PoolingRequestOutput from vllm import LLM, PoolingRequestOutput
from vllm.config import PoolerConfig from vllm.config import PoolerConfig
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.tasks import PoolingTask from vllm.tasks import PoolingTask
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach" MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
@@ -17,24 +16,21 @@ num_labels = 2
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def llm(): def llm(vllm_runner):
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection MODEL_NAME,
llm = LLM( max_model_len=None,
model=MODEL_NAME,
pooler_config=PoolerConfig(task="token_classify"), pooler_config=PoolerConfig(task="token_classify"),
max_num_batched_tokens=32768, max_num_batched_tokens=32768,
tensor_parallel_size=1, tensor_parallel_size=1,
gpu_memory_utilization=0.75, gpu_memory_utilization=0.75,
enforce_eager=True, enforce_eager=True,
seed=0, seed=0,
) enable_chunked_prefill=None,
) as runner:
yield weakref.proxy(llm) # pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
del llm yield weakref.proxy(runner.llm)
cleanup_dist_env_and_memory()
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@@ -6,7 +6,6 @@ import pytest
from vllm import LLM, PoolingRequestOutput from vllm import LLM, PoolingRequestOutput
from vllm.config import PoolerConfig from vllm.config import PoolerConfig
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.tasks import PoolingTask from vllm.tasks import PoolingTask
MODEL_NAME = "intfloat/multilingual-e5-small" MODEL_NAME = "intfloat/multilingual-e5-small"
@@ -17,24 +16,22 @@ embedding_size = 384
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def llm(): def llm(vllm_runner):
# pytest caches the fixture so we use weakref.proxy to with vllm_runner(
# enable garbage collection MODEL_NAME,
llm = LLM( max_model_len=None,
model=MODEL_NAME,
pooler_config=PoolerConfig(task="token_embed"), pooler_config=PoolerConfig(task="token_embed"),
max_num_batched_tokens=32768, max_num_batched_tokens=32768,
tensor_parallel_size=1, tensor_parallel_size=1,
gpu_memory_utilization=0.75, gpu_memory_utilization=0.75,
enforce_eager=True, enforce_eager=True,
seed=0, seed=0,
) enable_chunked_prefill=None,
assert embedding_size == llm.model_config.embedding_size ) as runner:
assert embedding_size == runner.llm.model_config.embedding_size
yield weakref.proxy(llm) # pytest caches yielded fixtures until after teardown, so use a proxy to
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
del llm yield weakref.proxy(runner.llm)
cleanup_dist_env_and_memory()
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@@ -8,13 +8,13 @@ actual NCCL communication.
""" """
import os import os
import weakref
from dataclasses import dataclass from dataclasses import dataclass
from unittest.mock import patch from unittest.mock import patch
import pytest import pytest
import torch import torch
from vllm import LLM
from vllm.config import WeightTransferConfig from vllm.config import WeightTransferConfig
from vllm.distributed.weight_transfer.base import ( from vllm.distributed.weight_transfer.base import (
WeightTransferEngine, WeightTransferEngine,
@@ -102,25 +102,24 @@ def mock_create_engine(config, vllm_config, device, model):
@create_new_process_for_each_test() @create_new_process_for_each_test()
def test_get_world_size_tp1(): def test_get_world_size_tp1(vllm_runner):
"""Test world_size is correctly configured for TP=1.""" """Test world_size is correctly configured for TP=1."""
if torch.accelerator.device_count() < 1: if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test") pytest.skip("Need at least 1 GPU for this test")
llm = LLM( with vllm_runner(
model=MODEL_NAME, MODEL_NAME,
enforce_eager=True, enforce_eager=True,
load_format="dummy", load_format="dummy",
tensor_parallel_size=1, tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"), weight_transfer_config=WeightTransferConfig(backend="nccl"),
) ) as runner:
world_size = runner.llm.llm_engine.vllm_config.parallel_config.world_size
world_size = llm.llm_engine.vllm_config.parallel_config.world_size assert world_size == 1
assert world_size == 1
@create_new_process_for_each_test() @create_new_process_for_each_test()
def test_init_weight_transfer_engine_calls_engine(): def test_init_weight_transfer_engine_calls_engine(vllm_runner):
"""Test that init_weight_transfer_engine calls the engine's """Test that init_weight_transfer_engine calls the engine's
init_transfer_engine method.""" init_transfer_engine method."""
if torch.accelerator.device_count() < 1: if torch.accelerator.device_count() < 1:
@@ -131,17 +130,20 @@ def test_init_weight_transfer_engine_calls_engine():
# Enable insecure serialization to allow pickling functions for collective_rpc # Enable insecure serialization to allow pickling functions for collective_rpc
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1" os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
with patch( with (
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine", patch(
mock_create_engine, "vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
): mock_create_engine,
llm = LLM( ),
model=MODEL_NAME, vllm_runner(
MODEL_NAME,
enforce_eager=True, enforce_eager=True,
load_format="dummy", load_format="dummy",
tensor_parallel_size=1, tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"), weight_transfer_config=WeightTransferConfig(backend="nccl"),
) ) as runner,
):
llm = weakref.proxy(runner.llm)
# Verify engine was created # Verify engine was created
def check_engine_exists(self): def check_engine_exists(self):
@@ -170,7 +172,7 @@ def test_init_weight_transfer_engine_calls_engine():
@create_new_process_for_each_test() @create_new_process_for_each_test()
def test_update_weights_calls_engine(): def test_update_weights_calls_engine(vllm_runner):
"""Test that update_weights calls the engine's receive_weights method.""" """Test that update_weights calls the engine's receive_weights method."""
if torch.accelerator.device_count() < 1: if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test") pytest.skip("Need at least 1 GPU for this test")
@@ -180,17 +182,20 @@ def test_update_weights_calls_engine():
# Enable insecure serialization to allow pickling functions for collective_rpc # Enable insecure serialization to allow pickling functions for collective_rpc
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1" os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
with patch( with (
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine", patch(
mock_create_engine, "vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
): mock_create_engine,
llm = LLM( ),
model=MODEL_NAME, vllm_runner(
MODEL_NAME,
enforce_eager=True, enforce_eager=True,
load_format="dummy", load_format="dummy",
tensor_parallel_size=1, tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"), weight_transfer_config=WeightTransferConfig(backend="nccl"),
) ) as runner,
):
llm = weakref.proxy(runner.llm)
# First init the weight transfer # First init the weight transfer
llm.init_weight_transfer_engine( llm.init_weight_transfer_engine(
@@ -233,7 +238,7 @@ def test_update_weights_calls_engine():
@create_new_process_for_each_test() @create_new_process_for_each_test()
def test_full_weight_transfer_flow(): def test_full_weight_transfer_flow(vllm_runner):
"""Test the complete weight transfer flow: init -> start -> update -> finish.""" """Test the complete weight transfer flow: init -> start -> update -> finish."""
if torch.accelerator.device_count() < 1: if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test") pytest.skip("Need at least 1 GPU for this test")
@@ -243,17 +248,20 @@ def test_full_weight_transfer_flow():
# Enable insecure serialization to allow pickling functions for collective_rpc # Enable insecure serialization to allow pickling functions for collective_rpc
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1" os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
with patch( with (
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine", patch(
mock_create_engine, "vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
): mock_create_engine,
llm = LLM( ),
model=MODEL_NAME, vllm_runner(
MODEL_NAME,
enforce_eager=True, enforce_eager=True,
load_format="dummy", load_format="dummy",
tensor_parallel_size=1, tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"), weight_transfer_config=WeightTransferConfig(backend="nccl"),
) ) as runner,
):
llm = weakref.proxy(runner.llm)
assert llm.get_weight_version() == "default" assert llm.get_weight_version() == "default"
@@ -309,20 +317,19 @@ def test_full_weight_transfer_flow():
@create_new_process_for_each_test() @create_new_process_for_each_test()
def test_weight_transfer_config_backend(): def test_weight_transfer_config_backend(vllm_runner):
"""Test that WeightTransferConfig backend is properly configured.""" """Test that WeightTransferConfig backend is properly configured."""
if torch.accelerator.device_count() < 1: if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test") pytest.skip("Need at least 1 GPU for this test")
# Test with nccl backend # Test with nccl backend
llm = LLM( with vllm_runner(
model=MODEL_NAME, MODEL_NAME,
enforce_eager=True, enforce_eager=True,
load_format="dummy", load_format="dummy",
tensor_parallel_size=1, tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"), weight_transfer_config=WeightTransferConfig(backend="nccl"),
) ) as runner:
config = runner.llm.llm_engine.vllm_config.weight_transfer_config
config = llm.llm_engine.vllm_config.weight_transfer_config assert config is not None
assert config is not None assert config.backend == "nccl"
assert config.backend == "nccl"