[ROCm][CI] Solidify entrypoint LLM lifecycle (#51280)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -9,9 +9,6 @@ import pytest
|
|||||||
import regex as re
|
import regex as re
|
||||||
import urllib3
|
import urllib3
|
||||||
|
|
||||||
from vllm import LLM
|
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
|
|
||||||
MODEL_CONFIGS = [
|
MODEL_CONFIGS = [
|
||||||
{
|
{
|
||||||
"model": "facebook/opt-125m",
|
"model": "facebook/opt-125m",
|
||||||
@@ -55,19 +52,26 @@ MODEL_CONFIGS = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _create_runner(vllm_runner, model_config):
|
||||||
|
runner_config = model_config.copy()
|
||||||
|
model = runner_config.pop("model")
|
||||||
|
tokenizer_name = runner_config.pop("tokenizer", None)
|
||||||
|
return vllm_runner(model, tokenizer_name=tokenizer_name, **runner_config)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def cache_models():
|
def cache_models(vllm_runner):
|
||||||
# Cache model files first
|
# Cache model files first
|
||||||
for model_config in MODEL_CONFIGS:
|
for model_config in MODEL_CONFIGS:
|
||||||
LLM(**model_config)
|
with _create_runner(vllm_runner, model_config):
|
||||||
cleanup_dist_env_and_memory()
|
pass
|
||||||
|
|
||||||
yield
|
yield
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_global_cleanup
|
@pytest.mark.skip_global_cleanup
|
||||||
@pytest.mark.usefixtures("cache_models")
|
@pytest.mark.usefixtures("cache_models")
|
||||||
def test_offline_mode(monkeypatch: pytest.MonkeyPatch):
|
def test_offline_mode(monkeypatch: pytest.MonkeyPatch, vllm_runner):
|
||||||
# Set HF to offline mode and ensure we can still construct an LLM
|
# Set HF to offline mode and ensure we can still construct an LLM
|
||||||
with monkeypatch.context() as m:
|
with monkeypatch.context() as m:
|
||||||
try:
|
try:
|
||||||
@@ -93,7 +97,8 @@ def test_offline_mode(monkeypatch: pytest.MonkeyPatch):
|
|||||||
_re_import_modules()
|
_re_import_modules()
|
||||||
# Cached model files should be used in offline mode
|
# Cached model files should be used in offline mode
|
||||||
for model_config in MODEL_CONFIGS:
|
for model_config in MODEL_CONFIGS:
|
||||||
LLM(**model_config)
|
with _create_runner(vllm_runner, model_config):
|
||||||
|
pass
|
||||||
finally:
|
finally:
|
||||||
# Reset the environment after the test
|
# Reset the environment after the test
|
||||||
# NB: Assuming tests are run in online mode
|
# NB: Assuming tests are run in online mode
|
||||||
@@ -136,7 +141,7 @@ def _re_import_modules():
|
|||||||
|
|
||||||
@pytest.mark.skip_global_cleanup
|
@pytest.mark.skip_global_cleanup
|
||||||
@pytest.mark.usefixtures("cache_models")
|
@pytest.mark.usefixtures("cache_models")
|
||||||
def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch):
|
def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch, vllm_runner):
|
||||||
# Set HF to offline mode and ensure we can still construct an LLM
|
# Set HF to offline mode and ensure we can still construct an LLM
|
||||||
with monkeypatch.context() as m:
|
with monkeypatch.context() as m:
|
||||||
try:
|
try:
|
||||||
@@ -159,7 +164,8 @@ def test_model_from_huggingface_offline(monkeypatch: pytest.MonkeyPatch):
|
|||||||
# Need to re-import huggingface_hub
|
# Need to re-import huggingface_hub
|
||||||
# and friends to set up offline mode
|
# and friends to set up offline mode
|
||||||
_re_import_modules()
|
_re_import_modules()
|
||||||
LLM(model="facebook/opt-125m")
|
with vllm_runner("facebook/opt-125m"):
|
||||||
|
pass
|
||||||
finally:
|
finally:
|
||||||
# Reset the environment after the test
|
# Reset the environment after the test
|
||||||
# NB: Assuming tests are run in online mode
|
# NB: Assuming tests are run in online mode
|
||||||
|
|||||||
@@ -4,46 +4,36 @@ import weakref
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm import LLM
|
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.sampling_params import SamplingParams
|
from vllm.sampling_params import SamplingParams
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="function")
|
@pytest.fixture(scope="function")
|
||||||
def text_llm():
|
def text_llm(vllm_runner):
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
"meta-llama/Llama-3.2-1B-Instruct", enforce_eager=True, seed=0
|
||||||
llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct", enforce_eager=True, seed=0)
|
) as runner:
|
||||||
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
yield weakref.proxy(llm)
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
|
yield weakref.proxy(runner.llm)
|
||||||
del llm
|
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="function")
|
@pytest.fixture(scope="function")
|
||||||
def llm_for_failure_test():
|
def llm_for_failure_test(vllm_runner):
|
||||||
"""
|
"""
|
||||||
Fixture for testing issue #26081.
|
Fixture for testing issue #26081.
|
||||||
Uses a small max_model_len to easily trigger length errors.
|
Uses a small max_model_len to easily trigger length errors.
|
||||||
"""
|
"""
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
"meta-llama/Llama-3.2-1B-Instruct",
|
||||||
llm = LLM(
|
|
||||||
model="meta-llama/Llama-3.2-1B-Instruct",
|
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
seed=0,
|
seed=0,
|
||||||
max_model_len=128,
|
max_model_len=128,
|
||||||
disable_log_stats=True,
|
disable_log_stats=True,
|
||||||
)
|
) as runner:
|
||||||
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
yield weakref.proxy(llm)
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
|
yield weakref.proxy(runner.llm)
|
||||||
del llm
|
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
def test_chat(text_llm):
|
def test_chat(text_llm):
|
||||||
@@ -99,21 +89,16 @@ def test_llm_chat_tokenization_no_double_bos(text_llm):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="function")
|
@pytest.fixture(scope="function")
|
||||||
def thinking_llm():
|
def thinking_llm(vllm_runner):
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
"Qwen/Qwen3-0.6B",
|
||||||
llm = LLM(
|
|
||||||
model="Qwen/Qwen3-0.6B",
|
|
||||||
max_model_len=4096,
|
max_model_len=4096,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
seed=0,
|
seed=0,
|
||||||
)
|
) as runner:
|
||||||
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
yield weakref.proxy(llm)
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
|
yield weakref.proxy(runner.llm)
|
||||||
del llm
|
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("enable_thinking", [True, False])
|
@pytest.mark.parametrize("enable_thinking", [True, False])
|
||||||
|
|||||||
@@ -4,15 +4,13 @@
|
|||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm import LLM
|
|
||||||
|
|
||||||
from ...utils import create_new_process_for_each_test
|
from ...utils import create_new_process_for_each_test
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||||
@pytest.mark.parametrize("backend", ["mp", "ray"])
|
@pytest.mark.parametrize("backend", ["mp", "ray"])
|
||||||
@create_new_process_for_each_test()
|
@create_new_process_for_each_test()
|
||||||
def test_collective_rpc(tp_size, backend, monkeypatch):
|
def test_collective_rpc(tp_size, backend, monkeypatch, vllm_runner):
|
||||||
if torch.accelerator.device_count() < tp_size:
|
if torch.accelerator.device_count() < tp_size:
|
||||||
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
|
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
|
||||||
if tp_size == 1 and backend == "ray":
|
if tp_size == 1 and backend == "ray":
|
||||||
@@ -26,11 +24,11 @@ def test_collective_rpc(tp_size, backend, monkeypatch):
|
|||||||
return self.rank
|
return self.rank
|
||||||
|
|
||||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||||
llm = LLM(
|
with vllm_runner(
|
||||||
model="hmellor/tiny-random-LlamaForCausalLM",
|
"hmellor/tiny-random-LlamaForCausalLM",
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
load_format="dummy",
|
load_format="dummy",
|
||||||
tensor_parallel_size=tp_size,
|
tensor_parallel_size=tp_size,
|
||||||
distributed_executor_backend=backend,
|
distributed_executor_backend=backend,
|
||||||
)
|
) as runner:
|
||||||
assert llm.collective_rpc(echo_rank) == list(range(tp_size))
|
assert runner.llm.collective_rpc(echo_rank) == list(range(tp_size))
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import weakref
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm import LLM, SamplingParams
|
from vllm import LLM, SamplingParams
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
|
|
||||||
MODEL_NAME = "distilbert/distilgpt2"
|
MODEL_NAME = "distilbert/distilgpt2"
|
||||||
|
|
||||||
@@ -26,22 +25,17 @@ TOKEN_IDS = [
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def llm():
|
def llm(vllm_runner):
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
MODEL_NAME,
|
||||||
llm = LLM(
|
|
||||||
model=MODEL_NAME,
|
|
||||||
max_num_batched_tokens=4096,
|
max_num_batched_tokens=4096,
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
gpu_memory_utilization=0.10,
|
gpu_memory_utilization=0.10,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
)
|
) as runner:
|
||||||
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
yield weakref.proxy(llm)
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
|
yield weakref.proxy(runner.llm)
|
||||||
del llm
|
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_global_cleanup
|
@pytest.mark.skip_global_cleanup
|
||||||
@@ -97,34 +91,34 @@ def test_single_prompt_priority(llm: LLM):
|
|||||||
assert len(outputs) == 1
|
assert len(outputs) == 1
|
||||||
|
|
||||||
|
|
||||||
def test_max_model_len():
|
def test_max_model_len(vllm_runner):
|
||||||
max_model_len = 20
|
max_model_len = 20
|
||||||
llm = LLM(
|
with vllm_runner(
|
||||||
model=MODEL_NAME,
|
MODEL_NAME,
|
||||||
max_model_len=max_model_len,
|
max_model_len=max_model_len,
|
||||||
gpu_memory_utilization=0.10,
|
gpu_memory_utilization=0.10,
|
||||||
enforce_eager=True, # reduce test time
|
enforce_eager=True, # reduce test time
|
||||||
)
|
) as runner:
|
||||||
sampling_params = SamplingParams(max_tokens=max_model_len + 10)
|
sampling_params = SamplingParams(max_tokens=max_model_len + 10)
|
||||||
outputs = llm.generate(PROMPTS, sampling_params)
|
outputs = runner.llm.generate(PROMPTS, sampling_params)
|
||||||
for output in outputs:
|
for output in outputs:
|
||||||
num_total_tokens = len(output.prompt_token_ids) + len(
|
num_total_tokens = len(output.prompt_token_ids) + len(
|
||||||
output.outputs[0].token_ids
|
output.outputs[0].token_ids
|
||||||
)
|
)
|
||||||
# Total tokens must not exceed max_model_len.
|
# Total tokens must not exceed max_model_len.
|
||||||
# It can be less if generation finishes due to other reasons (e.g., EOS)
|
# It can be less if generation finishes due to other reasons (e.g., EOS)
|
||||||
# before reaching the absolute model length limit.
|
# before reaching the absolute model length limit.
|
||||||
assert num_total_tokens <= max_model_len
|
assert num_total_tokens <= max_model_len
|
||||||
|
|
||||||
|
|
||||||
def test_log_stats():
|
def test_log_stats(vllm_runner):
|
||||||
llm = LLM(
|
with vllm_runner(
|
||||||
model=MODEL_NAME,
|
MODEL_NAME,
|
||||||
disable_log_stats=False,
|
disable_log_stats=False,
|
||||||
gpu_memory_utilization=0.10,
|
gpu_memory_utilization=0.10,
|
||||||
enforce_eager=True, # reduce test time
|
enforce_eager=True, # reduce test time
|
||||||
)
|
) as runner:
|
||||||
outputs = llm.generate(PROMPTS, sampling_params=None)
|
outputs = runner.llm.generate(PROMPTS, sampling_params=None)
|
||||||
|
|
||||||
# disable_log_stats is False, every output should have metrics
|
# disable_log_stats is False, every output should have metrics
|
||||||
assert all(output.metrics is not None for output in outputs)
|
assert all(output.metrics is not None for output in outputs)
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from vllm import LLM, SamplingParams
|
from vllm import SamplingParams
|
||||||
|
|
||||||
|
|
||||||
def test_gpu_memory_utilization():
|
def test_gpu_memory_utilization(vllm_runner):
|
||||||
prompts = [
|
prompts = [
|
||||||
"Hello, my name is",
|
"Hello, my name is",
|
||||||
"The president of the United States is",
|
"The president of the United States is",
|
||||||
@@ -15,13 +15,26 @@ def test_gpu_memory_utilization():
|
|||||||
|
|
||||||
# makes sure gpu_memory_utilization is per-instance limit,
|
# makes sure gpu_memory_utilization is per-instance limit,
|
||||||
# not a global limit
|
# not a global limit
|
||||||
llms = [
|
with (
|
||||||
LLM(model="facebook/opt-125m", gpu_memory_utilization=0.3, enforce_eager=True)
|
vllm_runner(
|
||||||
for i in range(3)
|
"facebook/opt-125m",
|
||||||
]
|
gpu_memory_utilization=0.3,
|
||||||
for llm in llms:
|
enforce_eager=True,
|
||||||
outputs = llm.generate(prompts, sampling_params)
|
) as runner_0,
|
||||||
for output in outputs:
|
vllm_runner(
|
||||||
prompt = output.prompt
|
"facebook/opt-125m",
|
||||||
generated_text = output.outputs[0].text
|
gpu_memory_utilization=0.3,
|
||||||
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
|
enforce_eager=True,
|
||||||
|
) as runner_1,
|
||||||
|
vllm_runner(
|
||||||
|
"facebook/opt-125m",
|
||||||
|
gpu_memory_utilization=0.3,
|
||||||
|
enforce_eager=True,
|
||||||
|
) as runner_2,
|
||||||
|
):
|
||||||
|
for runner in (runner_0, runner_1, runner_2):
|
||||||
|
outputs = runner.llm.generate(prompts, sampling_params)
|
||||||
|
for output in outputs:
|
||||||
|
prompt = output.prompt
|
||||||
|
generated_text = output.outputs[0].text
|
||||||
|
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
|
||||||
|
|||||||
@@ -4,30 +4,35 @@
|
|||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm import LLM
|
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
|
|
||||||
|
|
||||||
def test_empty_prompt():
|
def test_empty_prompt(vllm_runner):
|
||||||
llm = LLM(model="openai-community/gpt2", enforce_eager=True)
|
with (
|
||||||
with pytest.raises(VLLMValidationError, match="decoder prompt cannot be empty"):
|
vllm_runner("openai-community/gpt2", enforce_eager=True) as runner,
|
||||||
llm.generate([""])
|
pytest.raises(VLLMValidationError, match="decoder prompt cannot be empty"),
|
||||||
|
):
|
||||||
|
runner.llm.generate([""])
|
||||||
|
|
||||||
|
|
||||||
def test_out_of_vocab_token():
|
def test_out_of_vocab_token(vllm_runner):
|
||||||
llm = LLM(model="openai-community/gpt2", enforce_eager=True)
|
with (
|
||||||
with pytest.raises(VLLMValidationError, match="out of vocabulary"):
|
vllm_runner("openai-community/gpt2", enforce_eager=True) as runner,
|
||||||
llm.generate({"prompt_token_ids": [999999]})
|
pytest.raises(VLLMValidationError, match="out of vocabulary"),
|
||||||
|
):
|
||||||
|
runner.llm.generate({"prompt_token_ids": [999999]})
|
||||||
|
|
||||||
|
|
||||||
def test_require_mm_embeds():
|
def test_require_mm_embeds(vllm_runner):
|
||||||
llm = LLM(
|
with (
|
||||||
model="llava-hf/llava-1.5-7b-hf",
|
vllm_runner(
|
||||||
enforce_eager=True,
|
"llava-hf/llava-1.5-7b-hf",
|
||||||
enable_mm_embeds=False,
|
enforce_eager=True,
|
||||||
)
|
enable_mm_embeds=False,
|
||||||
with pytest.raises(ValueError, match="--enable-mm-embeds"):
|
) as runner,
|
||||||
llm.generate(
|
pytest.raises(ValueError, match="--enable-mm-embeds"),
|
||||||
|
):
|
||||||
|
runner.llm.generate(
|
||||||
{
|
{
|
||||||
"prompt": "<image>",
|
"prompt": "<image>",
|
||||||
"multi_modal_data": {"image": torch.empty(1, 1, 1)},
|
"multi_modal_data": {"image": torch.empty(1, 1, 1)},
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,11 +1,5 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from collections.abc import Callable, Iterator
|
|
||||||
from contextlib import contextmanager
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
|
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
|
||||||
TEST_IMAGE_ASSETS = [
|
TEST_IMAGE_ASSETS = [
|
||||||
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
|
||||||
@@ -13,70 +7,3 @@ TEST_IMAGE_ASSETS = [
|
|||||||
"1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
|
"1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
|
||||||
"RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
|
"RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def _shutdown_llm(llm: Any, gpu_memory_utilization: float) -> None:
|
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
from vllm.platforms import current_platform
|
|
||||||
|
|
||||||
try:
|
|
||||||
shutdown_timeout = 60.0 if current_platform.is_rocm() else None
|
|
||||||
llm.llm_engine.engine_core.shutdown(timeout=shutdown_timeout)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
del llm
|
|
||||||
|
|
||||||
try:
|
|
||||||
import torch
|
|
||||||
|
|
||||||
torch._dynamo.reset()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
if current_platform.is_rocm():
|
|
||||||
from tests.utils import wait_for_rocm_memory_to_settle
|
|
||||||
|
|
||||||
wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def managed_llm(*args: Any, **kwargs: Any) -> Iterator[Any]:
|
|
||||||
from vllm import LLM
|
|
||||||
|
|
||||||
llm = LLM(*args, **kwargs)
|
|
||||||
gpu_memory_utilization = (
|
|
||||||
llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
yield llm
|
|
||||||
finally:
|
|
||||||
_shutdown_llm(llm, gpu_memory_utilization)
|
|
||||||
|
|
||||||
|
|
||||||
def _make_managed_llm_factory() -> Iterator[Callable[..., Any]]:
|
|
||||||
from vllm import LLM
|
|
||||||
|
|
||||||
llms: list[tuple[Any, float]] = []
|
|
||||||
|
|
||||||
def make_llm(*args: Any, **kwargs: Any) -> Any:
|
|
||||||
llm = LLM(*args, **kwargs)
|
|
||||||
gpu_memory_utilization = (
|
|
||||||
llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
|
|
||||||
)
|
|
||||||
llms.append((llm, gpu_memory_utilization))
|
|
||||||
return llm
|
|
||||||
|
|
||||||
try:
|
|
||||||
yield make_llm
|
|
||||||
finally:
|
|
||||||
while llms:
|
|
||||||
llm, gpu_memory_utilization = llms.pop()
|
|
||||||
_shutdown_llm(llm, gpu_memory_utilization)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def multimodal_llm_factory() -> Iterator[Callable[..., Any]]:
|
|
||||||
yield from _make_managed_llm_factory()
|
|
||||||
|
|||||||
@@ -1,21 +1,26 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
import weakref
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
|
from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="function")
|
@pytest.fixture(scope="function")
|
||||||
def vision_llm(multimodal_llm_factory):
|
def vision_llm(vllm_runner):
|
||||||
return multimodal_llm_factory(
|
with vllm_runner(
|
||||||
model="microsoft/Phi-3.5-vision-instruct",
|
"microsoft/Phi-3.5-vision-instruct",
|
||||||
max_model_len=4096,
|
max_model_len=4096,
|
||||||
max_num_seqs=5,
|
max_num_seqs=5,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
limit_mm_per_prompt={"image": 2},
|
limit_mm_per_prompt={"image": 2},
|
||||||
seed=0,
|
seed=0,
|
||||||
)
|
) as runner:
|
||||||
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
|
yield weakref.proxy(runner.llm)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
|
|||||||
@@ -69,7 +69,7 @@ def test_inject_into_mm_cache(
|
|||||||
image_urls,
|
image_urls,
|
||||||
mm_processor_cache_type,
|
mm_processor_cache_type,
|
||||||
caplog_vllm,
|
caplog_vllm,
|
||||||
multimodal_llm_factory,
|
vllm_runner,
|
||||||
):
|
):
|
||||||
"""Test that inject_into_mm_cache() injects pre-processed mm_kwargs into
|
"""Test that inject_into_mm_cache() injects pre-processed mm_kwargs into
|
||||||
the processor cache and MM cache hit metrics are updated correctly.
|
the processor cache and MM cache hit metrics are updated correctly.
|
||||||
@@ -79,117 +79,115 @@ def test_inject_into_mm_cache(
|
|||||||
2. Extract cached kwargs, call inject_into_mm_cache with a new hash,
|
2. Extract cached kwargs, call inject_into_mm_cache with a new hash,
|
||||||
then generate with a pre-rendered input -> verifies injection works
|
then generate with a pre-rendered input -> verifies injection works
|
||||||
"""
|
"""
|
||||||
llm = multimodal_llm_factory(
|
with vllm_runner(
|
||||||
model="llava-hf/llava-1.5-7b-hf",
|
"llava-hf/llava-1.5-7b-hf",
|
||||||
max_model_len=4096,
|
max_model_len=4096,
|
||||||
max_num_seqs=5,
|
max_num_seqs=5,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
disable_log_stats=False,
|
disable_log_stats=False,
|
||||||
limit_mm_per_prompt={"image": 2},
|
limit_mm_per_prompt={"image": 2},
|
||||||
mm_processor_cache_type=mm_processor_cache_type,
|
mm_processor_cache_type=mm_processor_cache_type,
|
||||||
)
|
) as runner:
|
||||||
|
# Step 1: Normal requests to populate the cache
|
||||||
|
runner.llm.chat(_make_messages(image_urls[0]))
|
||||||
|
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
|
||||||
|
|
||||||
# Step 1: Normal requests to populate the cache
|
runner.llm.chat(_make_messages(image_urls[0]))
|
||||||
llm.chat(_make_messages(image_urls[0]))
|
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 1)
|
||||||
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
|
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(50.0)
|
||||||
|
|
||||||
llm.chat(_make_messages(image_urls[0]))
|
# Step 2: Use a second image to get valid expanded tokens and
|
||||||
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 1)
|
# placeholder positions via the renderer.
|
||||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(50.0)
|
runner.llm.chat(_make_messages(image_urls[1]))
|
||||||
|
queries_before = _get_mm_cache_stats(runner.llm.get_metrics())[0] # 3
|
||||||
|
|
||||||
# Step 2: Use a second image to get valid expanded tokens and
|
renderer = runner.llm.llm_engine.renderer
|
||||||
# placeholder positions via the renderer.
|
cache = renderer.mm_processor_cache
|
||||||
llm.chat(_make_messages(image_urls[1]))
|
assert cache is not None, "Processor cache should be enabled"
|
||||||
queries_before = _get_mm_cache_stats(llm.get_metrics())[0] # 3
|
|
||||||
|
|
||||||
renderer = llm.llm_engine.renderer
|
_, eng_prompts = renderer.render_chat(
|
||||||
cache = renderer.mm_processor_cache
|
[_make_messages(image_urls[1])],
|
||||||
assert cache is not None, "Processor cache should be enabled"
|
ChatParams(),
|
||||||
|
)
|
||||||
|
eng_input = eng_prompts[0]
|
||||||
|
|
||||||
_, eng_prompts = renderer.render_chat(
|
# Inject pre-processed mm_kwargs with a NEW hash via public API
|
||||||
[_make_messages(image_urls[1])],
|
new_mm_hash = "deadbeef" * 8
|
||||||
ChatParams(),
|
mm_hashes = {"image": [new_mm_hash]}
|
||||||
)
|
mm_kwargs = eng_input["mm_kwargs"]
|
||||||
eng_input = eng_prompts[0]
|
|
||||||
|
|
||||||
# Inject pre-processed mm_kwargs with a NEW hash via public API
|
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
|
||||||
new_mm_hash = "deadbeef" * 8
|
|
||||||
mm_hashes = {"image": [new_mm_hash]}
|
|
||||||
mm_kwargs = eng_input["mm_kwargs"]
|
|
||||||
|
|
||||||
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
|
# Build pre-rendered input (no externally_processed flag needed)
|
||||||
|
pre_rendered_input = {
|
||||||
|
"type": "multimodal",
|
||||||
|
"prompt_token_ids": eng_input["prompt_token_ids"],
|
||||||
|
"mm_kwargs": mm_kwargs,
|
||||||
|
"mm_hashes": mm_hashes,
|
||||||
|
"mm_placeholders": eng_input["mm_placeholders"],
|
||||||
|
}
|
||||||
|
|
||||||
# Build pre-rendered input (no externally_processed flag needed)
|
runner.llm.generate(
|
||||||
pre_rendered_input = {
|
pre_rendered_input,
|
||||||
"type": "multimodal",
|
sampling_params=SamplingParams(max_tokens=1),
|
||||||
"prompt_token_ids": eng_input["prompt_token_ids"],
|
)
|
||||||
"mm_kwargs": mm_kwargs,
|
|
||||||
"mm_hashes": mm_hashes,
|
|
||||||
"mm_placeholders": eng_input["mm_placeholders"],
|
|
||||||
}
|
|
||||||
|
|
||||||
llm.generate(
|
# Verify cache was queried and injection happened
|
||||||
pre_rendered_input,
|
queries_after = _get_mm_cache_stats(runner.llm.get_metrics())[0]
|
||||||
sampling_params=SamplingParams(max_tokens=1),
|
assert queries_after > queries_before, (
|
||||||
)
|
"Cache should have been queried for the injected item"
|
||||||
|
)
|
||||||
# Verify cache was queried and injection happened
|
mm_rate = _get_mm_cache_log(runner.llm, caplog_vllm)
|
||||||
queries_after = _get_mm_cache_stats(llm.get_metrics())[0]
|
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
|
||||||
assert queries_after > queries_before, (
|
|
||||||
"Cache should have been queried for the injected item"
|
|
||||||
)
|
|
||||||
mm_rate = _get_mm_cache_log(llm, caplog_vllm)
|
|
||||||
assert mm_rate >= 0.0, "MM cache hit rate should be reported"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:1]], indirect=True)
|
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:1]], indirect=True)
|
||||||
def test_inject_into_mm_cache_without_cache(
|
def test_inject_into_mm_cache_without_cache(
|
||||||
num_gpus_available,
|
num_gpus_available,
|
||||||
image_urls,
|
image_urls,
|
||||||
multimodal_llm_factory,
|
vllm_runner,
|
||||||
):
|
):
|
||||||
"""Test that inject_into_mm_cache works gracefully when processor cache
|
"""Test that inject_into_mm_cache works gracefully when processor cache
|
||||||
is disabled (mm_processor_cache_gb=0). Should not crash.
|
is disabled (mm_processor_cache_gb=0). Should not crash.
|
||||||
"""
|
"""
|
||||||
llm = multimodal_llm_factory(
|
with vllm_runner(
|
||||||
model="llava-hf/llava-1.5-7b-hf",
|
"llava-hf/llava-1.5-7b-hf",
|
||||||
max_model_len=4096,
|
max_model_len=4096,
|
||||||
max_num_seqs=5,
|
max_num_seqs=5,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
disable_log_stats=False,
|
disable_log_stats=False,
|
||||||
limit_mm_per_prompt={"image": 2},
|
limit_mm_per_prompt={"image": 2},
|
||||||
mm_processor_cache_gb=0,
|
mm_processor_cache_gb=0,
|
||||||
)
|
) as runner:
|
||||||
|
# Run a normal chat request first to warm up the model.
|
||||||
|
runner.llm.chat(_make_messages(image_urls[0]))
|
||||||
|
|
||||||
# Run a normal chat request first to warm up the model.
|
# Use the renderer to get a proper EngineInput with expanded tokens
|
||||||
llm.chat(_make_messages(image_urls[0]))
|
renderer = runner.llm.llm_engine.renderer
|
||||||
|
_, eng_prompts = renderer.render_chat(
|
||||||
|
[_make_messages(image_urls[0])],
|
||||||
|
ChatParams(),
|
||||||
|
)
|
||||||
|
eng_input = eng_prompts[0]
|
||||||
|
|
||||||
# Use the renderer to get a proper EngineInput with expanded tokens
|
mm_hashes = {"image": ["abcd1234" * 8]}
|
||||||
renderer = llm.llm_engine.renderer
|
mm_kwargs = eng_input["mm_kwargs"]
|
||||||
_, eng_prompts = renderer.render_chat(
|
|
||||||
[_make_messages(image_urls[0])],
|
|
||||||
ChatParams(),
|
|
||||||
)
|
|
||||||
eng_input = eng_prompts[0]
|
|
||||||
|
|
||||||
mm_hashes = {"image": ["abcd1234" * 8]}
|
# inject_into_mm_cache should not crash even without cache
|
||||||
mm_kwargs = eng_input["mm_kwargs"]
|
runner.llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
|
||||||
|
|
||||||
# inject_into_mm_cache should not crash even without cache
|
# Build and generate with pre-rendered input
|
||||||
llm.llm_engine.input_processor.inject_into_mm_cache(mm_hashes, mm_kwargs)
|
pre_rendered_input = {
|
||||||
|
"type": "multimodal",
|
||||||
|
"prompt_token_ids": eng_input["prompt_token_ids"],
|
||||||
|
"mm_kwargs": mm_kwargs,
|
||||||
|
"mm_hashes": mm_hashes,
|
||||||
|
"mm_placeholders": eng_input["mm_placeholders"],
|
||||||
|
}
|
||||||
|
|
||||||
# Build and generate with pre-rendered input
|
result = runner.llm.generate(
|
||||||
pre_rendered_input = {
|
pre_rendered_input,
|
||||||
"type": "multimodal",
|
sampling_params=SamplingParams(max_tokens=1),
|
||||||
"prompt_token_ids": eng_input["prompt_token_ids"],
|
)
|
||||||
"mm_kwargs": mm_kwargs,
|
assert len(result) == 1, "Should produce one output"
|
||||||
"mm_hashes": mm_hashes,
|
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
|
||||||
"mm_placeholders": eng_input["mm_placeholders"],
|
|
||||||
}
|
|
||||||
|
|
||||||
result = llm.generate(
|
|
||||||
pre_rendered_input,
|
|
||||||
sampling_params=SamplingParams(max_tokens=1),
|
|
||||||
)
|
|
||||||
assert len(result) == 1, "Should produce one output"
|
|
||||||
assert len(result[0].outputs) >= 1, "Should have at least one output sequence"
|
|
||||||
|
|||||||
@@ -61,38 +61,37 @@ def test_mm_cache_stats(
|
|||||||
image_urls,
|
image_urls,
|
||||||
mm_processor_cache_type,
|
mm_processor_cache_type,
|
||||||
caplog_vllm,
|
caplog_vllm,
|
||||||
multimodal_llm_factory,
|
vllm_runner,
|
||||||
):
|
):
|
||||||
llm = multimodal_llm_factory(
|
with vllm_runner(
|
||||||
model="llava-hf/llava-1.5-7b-hf",
|
"llava-hf/llava-1.5-7b-hf",
|
||||||
max_model_len=4096,
|
max_model_len=4096,
|
||||||
max_num_seqs=5,
|
max_num_seqs=5,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
mm_processor_cache_type=mm_processor_cache_type,
|
mm_processor_cache_type=mm_processor_cache_type,
|
||||||
disable_log_stats=False,
|
disable_log_stats=False,
|
||||||
limit_mm_per_prompt={"image": 2},
|
limit_mm_per_prompt={"image": 2},
|
||||||
)
|
) as runner:
|
||||||
|
runner.llm.chat(_make_messages(image_urls[0]))
|
||||||
|
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (1, 0)
|
||||||
|
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
|
||||||
|
|
||||||
llm.chat(_make_messages(image_urls[0]))
|
runner.llm.chat(_make_messages(image_urls[1]))
|
||||||
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
|
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (2, 0)
|
||||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
|
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
|
||||||
|
|
||||||
llm.chat(_make_messages(image_urls[1]))
|
runner.llm.chat(_make_messages(image_urls[0]))
|
||||||
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 0)
|
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (3, 1)
|
||||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
|
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(33.3)
|
||||||
|
|
||||||
llm.chat(_make_messages(image_urls[0]))
|
# NOTE: This only resets hit rate stats in CachingMetrics
|
||||||
assert _get_mm_cache_stats(llm.get_metrics()) == (3, 1)
|
# The raw queries and hits counts remain unaffected
|
||||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(33.3)
|
runner.llm.reset_mm_cache()
|
||||||
|
|
||||||
# NOTE: This only resets hit rate stats in CachingMetrics
|
runner.llm.chat(_make_messages(image_urls[0]))
|
||||||
# The raw queries and hits counts remain unaffected
|
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (4, 1)
|
||||||
llm.reset_mm_cache()
|
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
|
||||||
|
|
||||||
llm.chat(_make_messages(image_urls[0]))
|
runner.llm.chat(_make_messages(image_urls[1]))
|
||||||
assert _get_mm_cache_stats(llm.get_metrics()) == (4, 1)
|
assert _get_mm_cache_stats(runner.llm.get_metrics()) == (5, 1)
|
||||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
|
assert _get_mm_cache_log(runner.llm, caplog_vllm) == pytest.approx(0.0)
|
||||||
|
|
||||||
llm.chat(_make_messages(image_urls[1]))
|
|
||||||
assert _get_mm_cache_stats(llm.get_metrics()) == (5, 1)
|
|
||||||
assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
|
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import weakref
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from tests.entrypoints.multimodal.conftest import managed_llm
|
|
||||||
from vllm import LLM, SamplingParams
|
from vllm import LLM, SamplingParams
|
||||||
from vllm.assets.image import ImageAsset
|
from vllm.assets.image import ImageAsset
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
@@ -14,17 +15,19 @@ TEXT_ONLY_PROMPT = "USER: What is 2 + 2?\nASSISTANT:"
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def llm():
|
def llm(vllm_runner):
|
||||||
"""LLM with enable_mm_embeds=True and all modality limits zeroed out."""
|
"""LLM with enable_mm_embeds=True and all modality limits zeroed out."""
|
||||||
with managed_llm(
|
with vllm_runner(
|
||||||
model=MODEL,
|
MODEL,
|
||||||
max_model_len=2048,
|
max_model_len=2048,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
gpu_memory_utilization=0.8,
|
gpu_memory_utilization=0.8,
|
||||||
enable_mm_embeds=True,
|
enable_mm_embeds=True,
|
||||||
limit_mm_per_prompt={"image": 0},
|
limit_mm_per_prompt={"image": 0},
|
||||||
) as llm:
|
) as runner:
|
||||||
yield llm
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
|
yield weakref.proxy(runner.llm)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_global_cleanup
|
@pytest.mark.skip_global_cleanup
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import weakref
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm import LLM, PoolingParams
|
from vllm import LLM, PoolingParams
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
|
|
||||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||||
@@ -29,23 +28,20 @@ TOKEN_IDS = [
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def llm():
|
def llm(vllm_runner):
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
MODEL_NAME,
|
||||||
llm = LLM(
|
max_model_len=None,
|
||||||
model=MODEL_NAME,
|
|
||||||
max_num_batched_tokens=32768,
|
max_num_batched_tokens=32768,
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
gpu_memory_utilization=0.75,
|
gpu_memory_utilization=0.75,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
seed=0,
|
seed=0,
|
||||||
)
|
enable_chunked_prefill=None,
|
||||||
|
) as runner:
|
||||||
yield weakref.proxy(llm)
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
del llm
|
yield weakref.proxy(runner.llm)
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_global_cleanup
|
@pytest.mark.skip_global_cleanup
|
||||||
|
|||||||
@@ -6,28 +6,26 @@ from unittest import mock
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm import LLM, PoolingParams
|
from vllm import PoolingParams
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
|
|
||||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def llm():
|
def llm(vllm_runner):
|
||||||
llm = LLM(
|
with vllm_runner(
|
||||||
model=MODEL_NAME,
|
MODEL_NAME,
|
||||||
|
max_model_len=None,
|
||||||
max_num_seqs=2, # small to trigger tiling
|
max_num_seqs=2, # small to trigger tiling
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
gpu_memory_utilization=0.75,
|
gpu_memory_utilization=0.75,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
seed=0,
|
seed=0,
|
||||||
)
|
enable_chunked_prefill=None,
|
||||||
|
) as runner:
|
||||||
yield weakref.proxy(llm)
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
del llm
|
yield weakref.proxy(runner.llm)
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_global_cleanup
|
@pytest.mark.skip_global_cleanup
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import torch
|
|||||||
|
|
||||||
from tests.models.utils import softmax
|
from tests.models.utils import softmax
|
||||||
from vllm import LLM, ClassificationRequestOutput, PoolingParams
|
from vllm import LLM, ClassificationRequestOutput, PoolingParams
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
from vllm.tasks import PoolingTask
|
from vllm.tasks import PoolingTask
|
||||||
|
|
||||||
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
|
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
|
||||||
@@ -18,23 +17,20 @@ num_labels = 2
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def llm():
|
def llm(vllm_runner):
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
MODEL_NAME,
|
||||||
llm = LLM(
|
max_model_len=None,
|
||||||
model=MODEL_NAME,
|
|
||||||
max_num_batched_tokens=32768,
|
max_num_batched_tokens=32768,
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
gpu_memory_utilization=0.75,
|
gpu_memory_utilization=0.75,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
seed=0,
|
seed=0,
|
||||||
)
|
enable_chunked_prefill=None,
|
||||||
|
) as runner:
|
||||||
yield weakref.proxy(llm)
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
del llm
|
yield weakref.proxy(runner.llm)
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_global_cleanup
|
@pytest.mark.skip_global_cleanup
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import torch
|
|||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
|
|
||||||
from vllm import LLM, EmbeddingRequestOutput, PoolingParams
|
from vllm import LLM, EmbeddingRequestOutput, PoolingParams
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
from vllm.tasks import PoolingTask
|
from vllm.tasks import PoolingTask
|
||||||
|
|
||||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||||
@@ -18,23 +17,21 @@ embedding_size = 384
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def llm():
|
def llm(vllm_runner):
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
MODEL_NAME,
|
||||||
llm = LLM(
|
max_model_len=None,
|
||||||
model=MODEL_NAME,
|
|
||||||
max_num_batched_tokens=32768,
|
max_num_batched_tokens=32768,
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
gpu_memory_utilization=0.75,
|
gpu_memory_utilization=0.75,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
seed=0,
|
seed=0,
|
||||||
)
|
enable_chunked_prefill=None,
|
||||||
assert embedding_size == llm.model_config.embedding_size
|
) as runner:
|
||||||
|
assert embedding_size == runner.llm.model_config.embedding_size
|
||||||
yield weakref.proxy(llm)
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
del llm
|
yield weakref.proxy(runner.llm)
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_global_cleanup
|
@pytest.mark.skip_global_cleanup
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import torch
|
|||||||
|
|
||||||
from tests.models.utils import softmax
|
from tests.models.utils import softmax
|
||||||
from vllm import LLM, PoolingParams
|
from vllm import LLM, PoolingParams
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
|
|
||||||
MODEL_NAME = "internlm/internlm2-1_8b-reward"
|
MODEL_NAME = "internlm/internlm2-1_8b-reward"
|
||||||
|
|
||||||
@@ -16,24 +15,21 @@ prompts = ["The chef prepared a delicious meal."]
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def llm():
|
def llm(vllm_runner):
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
MODEL_NAME,
|
||||||
llm = LLM(
|
max_model_len=None,
|
||||||
model=MODEL_NAME,
|
|
||||||
max_num_batched_tokens=32768,
|
max_num_batched_tokens=32768,
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
gpu_memory_utilization=0.75,
|
gpu_memory_utilization=0.75,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
seed=0,
|
seed=0,
|
||||||
)
|
enable_chunked_prefill=None,
|
||||||
|
) as runner:
|
||||||
yield weakref.proxy(llm)
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
del llm
|
yield weakref.proxy(runner.llm)
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_global_cleanup
|
@pytest.mark.skip_global_cleanup
|
||||||
|
|||||||
@@ -6,8 +6,6 @@ import weakref
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner
|
from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner
|
||||||
from vllm import LLM
|
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
|
|
||||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||||
PROMPT = "The chef prepared a delicious meal."
|
PROMPT = "The chef prepared a delicious meal."
|
||||||
@@ -27,23 +25,20 @@ DTYPE = "half"
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def llm():
|
def llm(vllm_runner):
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
MODEL_NAME,
|
||||||
llm = LLM(
|
max_model_len=None,
|
||||||
model=MODEL_NAME,
|
|
||||||
max_num_batched_tokens=32768,
|
max_num_batched_tokens=32768,
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
gpu_memory_utilization=0.75,
|
gpu_memory_utilization=0.75,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
seed=0,
|
seed=0,
|
||||||
)
|
enable_chunked_prefill=None,
|
||||||
|
) as runner:
|
||||||
yield weakref.proxy(llm)
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
del llm
|
yield weakref.proxy(runner.llm)
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import torch
|
|||||||
|
|
||||||
from tests.models.utils import softmax
|
from tests.models.utils import softmax
|
||||||
from vllm import LLM, PoolingParams
|
from vllm import LLM, PoolingParams
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
|
|
||||||
MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
|
MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
|
||||||
PROMPT = "The chef prepared a delicious meal."
|
PROMPT = "The chef prepared a delicious meal."
|
||||||
@@ -24,23 +23,20 @@ TEXTS_2 = [
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def llm():
|
def llm(vllm_runner):
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
MODEL_NAME,
|
||||||
llm = LLM(
|
max_model_len=None,
|
||||||
model=MODEL_NAME,
|
|
||||||
max_num_batched_tokens=32768,
|
max_num_batched_tokens=32768,
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
gpu_memory_utilization=0.75,
|
gpu_memory_utilization=0.75,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
seed=0,
|
seed=0,
|
||||||
)
|
enable_chunked_prefill=None,
|
||||||
|
) as runner:
|
||||||
yield weakref.proxy(llm)
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
del llm
|
yield weakref.proxy(runner.llm)
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
|
|||||||
@@ -5,9 +5,6 @@ import weakref
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm import LLM
|
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
|
|
||||||
from .util import ColBERTScoringHfRunner
|
from .util import ColBERTScoringHfRunner
|
||||||
|
|
||||||
MODEL_NAME = "answerdotai/answerai-colbert-small-v1"
|
MODEL_NAME = "answerdotai/answerai-colbert-small-v1"
|
||||||
@@ -30,23 +27,20 @@ DTYPE = "half"
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def llm():
|
def llm(vllm_runner):
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
MODEL_NAME,
|
||||||
llm = LLM(
|
max_model_len=None,
|
||||||
model=MODEL_NAME,
|
|
||||||
max_num_batched_tokens=32768,
|
max_num_batched_tokens=32768,
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
gpu_memory_utilization=0.75,
|
gpu_memory_utilization=0.75,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
seed=0,
|
seed=0,
|
||||||
)
|
enable_chunked_prefill=None,
|
||||||
|
) as runner:
|
||||||
yield weakref.proxy(llm)
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
del llm
|
yield weakref.proxy(runner.llm)
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
|
|||||||
@@ -5,32 +5,26 @@ import weakref
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm import LLM
|
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
|
|
||||||
from .util import make_base64_image, make_image_mm_param
|
from .util import make_base64_image, make_image_mm_param
|
||||||
|
|
||||||
MODEL_NAME = "vidore/colpali-v1.3-hf"
|
MODEL_NAME = "vidore/colpali-v1.3-hf"
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def llm():
|
def llm(vllm_runner):
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
MODEL_NAME,
|
||||||
llm = LLM(
|
max_model_len=None,
|
||||||
model=MODEL_NAME,
|
|
||||||
max_num_batched_tokens=32768,
|
max_num_batched_tokens=32768,
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
gpu_memory_utilization=0.75,
|
gpu_memory_utilization=0.75,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
seed=0,
|
seed=0,
|
||||||
)
|
enable_chunked_prefill=None,
|
||||||
|
) as runner:
|
||||||
yield weakref.proxy(llm)
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
del llm
|
yield weakref.proxy(runner.llm)
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_global_cleanup
|
@pytest.mark.skip_global_cleanup
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import pytest
|
|||||||
|
|
||||||
from vllm import LLM, PoolingRequestOutput
|
from vllm import LLM, PoolingRequestOutput
|
||||||
from vllm.config import PoolerConfig
|
from vllm.config import PoolerConfig
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
from vllm.tasks import PoolingTask
|
from vllm.tasks import PoolingTask
|
||||||
|
|
||||||
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
|
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
|
||||||
@@ -17,24 +16,21 @@ num_labels = 2
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def llm():
|
def llm(vllm_runner):
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
MODEL_NAME,
|
||||||
llm = LLM(
|
max_model_len=None,
|
||||||
model=MODEL_NAME,
|
|
||||||
pooler_config=PoolerConfig(task="token_classify"),
|
pooler_config=PoolerConfig(task="token_classify"),
|
||||||
max_num_batched_tokens=32768,
|
max_num_batched_tokens=32768,
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
gpu_memory_utilization=0.75,
|
gpu_memory_utilization=0.75,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
seed=0,
|
seed=0,
|
||||||
)
|
enable_chunked_prefill=None,
|
||||||
|
) as runner:
|
||||||
yield weakref.proxy(llm)
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
del llm
|
yield weakref.proxy(runner.llm)
|
||||||
|
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_global_cleanup
|
@pytest.mark.skip_global_cleanup
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import pytest
|
|||||||
|
|
||||||
from vllm import LLM, PoolingRequestOutput
|
from vllm import LLM, PoolingRequestOutput
|
||||||
from vllm.config import PoolerConfig
|
from vllm.config import PoolerConfig
|
||||||
from vllm.distributed import cleanup_dist_env_and_memory
|
|
||||||
from vllm.tasks import PoolingTask
|
from vllm.tasks import PoolingTask
|
||||||
|
|
||||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||||
@@ -17,24 +16,22 @@ embedding_size = 384
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def llm():
|
def llm(vllm_runner):
|
||||||
# pytest caches the fixture so we use weakref.proxy to
|
with vllm_runner(
|
||||||
# enable garbage collection
|
MODEL_NAME,
|
||||||
llm = LLM(
|
max_model_len=None,
|
||||||
model=MODEL_NAME,
|
|
||||||
pooler_config=PoolerConfig(task="token_embed"),
|
pooler_config=PoolerConfig(task="token_embed"),
|
||||||
max_num_batched_tokens=32768,
|
max_num_batched_tokens=32768,
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
gpu_memory_utilization=0.75,
|
gpu_memory_utilization=0.75,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
seed=0,
|
seed=0,
|
||||||
)
|
enable_chunked_prefill=None,
|
||||||
assert embedding_size == llm.model_config.embedding_size
|
) as runner:
|
||||||
|
assert embedding_size == runner.llm.model_config.embedding_size
|
||||||
yield weakref.proxy(llm)
|
# pytest caches yielded fixtures until after teardown, so use a proxy to
|
||||||
|
# avoid retaining the LLM while VllmRunner.__exit__ releases ROCm memory.
|
||||||
del llm
|
yield weakref.proxy(runner.llm)
|
||||||
cleanup_dist_env_and_memory()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_global_cleanup
|
@pytest.mark.skip_global_cleanup
|
||||||
|
|||||||
@@ -8,13 +8,13 @@ actual NCCL communication.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import weakref
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm import LLM
|
|
||||||
from vllm.config import WeightTransferConfig
|
from vllm.config import WeightTransferConfig
|
||||||
from vllm.distributed.weight_transfer.base import (
|
from vllm.distributed.weight_transfer.base import (
|
||||||
WeightTransferEngine,
|
WeightTransferEngine,
|
||||||
@@ -102,25 +102,24 @@ def mock_create_engine(config, vllm_config, device, model):
|
|||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test()
|
@create_new_process_for_each_test()
|
||||||
def test_get_world_size_tp1():
|
def test_get_world_size_tp1(vllm_runner):
|
||||||
"""Test world_size is correctly configured for TP=1."""
|
"""Test world_size is correctly configured for TP=1."""
|
||||||
if torch.accelerator.device_count() < 1:
|
if torch.accelerator.device_count() < 1:
|
||||||
pytest.skip("Need at least 1 GPU for this test")
|
pytest.skip("Need at least 1 GPU for this test")
|
||||||
|
|
||||||
llm = LLM(
|
with vllm_runner(
|
||||||
model=MODEL_NAME,
|
MODEL_NAME,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
load_format="dummy",
|
load_format="dummy",
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||||
)
|
) as runner:
|
||||||
|
world_size = runner.llm.llm_engine.vllm_config.parallel_config.world_size
|
||||||
world_size = llm.llm_engine.vllm_config.parallel_config.world_size
|
assert world_size == 1
|
||||||
assert world_size == 1
|
|
||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test()
|
@create_new_process_for_each_test()
|
||||||
def test_init_weight_transfer_engine_calls_engine():
|
def test_init_weight_transfer_engine_calls_engine(vllm_runner):
|
||||||
"""Test that init_weight_transfer_engine calls the engine's
|
"""Test that init_weight_transfer_engine calls the engine's
|
||||||
init_transfer_engine method."""
|
init_transfer_engine method."""
|
||||||
if torch.accelerator.device_count() < 1:
|
if torch.accelerator.device_count() < 1:
|
||||||
@@ -131,17 +130,20 @@ def test_init_weight_transfer_engine_calls_engine():
|
|||||||
# Enable insecure serialization to allow pickling functions for collective_rpc
|
# Enable insecure serialization to allow pickling functions for collective_rpc
|
||||||
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
||||||
|
|
||||||
with patch(
|
with (
|
||||||
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
|
patch(
|
||||||
mock_create_engine,
|
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
|
||||||
):
|
mock_create_engine,
|
||||||
llm = LLM(
|
),
|
||||||
model=MODEL_NAME,
|
vllm_runner(
|
||||||
|
MODEL_NAME,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
load_format="dummy",
|
load_format="dummy",
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||||
)
|
) as runner,
|
||||||
|
):
|
||||||
|
llm = weakref.proxy(runner.llm)
|
||||||
|
|
||||||
# Verify engine was created
|
# Verify engine was created
|
||||||
def check_engine_exists(self):
|
def check_engine_exists(self):
|
||||||
@@ -170,7 +172,7 @@ def test_init_weight_transfer_engine_calls_engine():
|
|||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test()
|
@create_new_process_for_each_test()
|
||||||
def test_update_weights_calls_engine():
|
def test_update_weights_calls_engine(vllm_runner):
|
||||||
"""Test that update_weights calls the engine's receive_weights method."""
|
"""Test that update_weights calls the engine's receive_weights method."""
|
||||||
if torch.accelerator.device_count() < 1:
|
if torch.accelerator.device_count() < 1:
|
||||||
pytest.skip("Need at least 1 GPU for this test")
|
pytest.skip("Need at least 1 GPU for this test")
|
||||||
@@ -180,17 +182,20 @@ def test_update_weights_calls_engine():
|
|||||||
# Enable insecure serialization to allow pickling functions for collective_rpc
|
# Enable insecure serialization to allow pickling functions for collective_rpc
|
||||||
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
||||||
|
|
||||||
with patch(
|
with (
|
||||||
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
|
patch(
|
||||||
mock_create_engine,
|
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
|
||||||
):
|
mock_create_engine,
|
||||||
llm = LLM(
|
),
|
||||||
model=MODEL_NAME,
|
vllm_runner(
|
||||||
|
MODEL_NAME,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
load_format="dummy",
|
load_format="dummy",
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||||
)
|
) as runner,
|
||||||
|
):
|
||||||
|
llm = weakref.proxy(runner.llm)
|
||||||
|
|
||||||
# First init the weight transfer
|
# First init the weight transfer
|
||||||
llm.init_weight_transfer_engine(
|
llm.init_weight_transfer_engine(
|
||||||
@@ -233,7 +238,7 @@ def test_update_weights_calls_engine():
|
|||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test()
|
@create_new_process_for_each_test()
|
||||||
def test_full_weight_transfer_flow():
|
def test_full_weight_transfer_flow(vllm_runner):
|
||||||
"""Test the complete weight transfer flow: init -> start -> update -> finish."""
|
"""Test the complete weight transfer flow: init -> start -> update -> finish."""
|
||||||
if torch.accelerator.device_count() < 1:
|
if torch.accelerator.device_count() < 1:
|
||||||
pytest.skip("Need at least 1 GPU for this test")
|
pytest.skip("Need at least 1 GPU for this test")
|
||||||
@@ -243,17 +248,20 @@ def test_full_weight_transfer_flow():
|
|||||||
# Enable insecure serialization to allow pickling functions for collective_rpc
|
# Enable insecure serialization to allow pickling functions for collective_rpc
|
||||||
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
||||||
|
|
||||||
with patch(
|
with (
|
||||||
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
|
patch(
|
||||||
mock_create_engine,
|
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
|
||||||
):
|
mock_create_engine,
|
||||||
llm = LLM(
|
),
|
||||||
model=MODEL_NAME,
|
vllm_runner(
|
||||||
|
MODEL_NAME,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
load_format="dummy",
|
load_format="dummy",
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||||
)
|
) as runner,
|
||||||
|
):
|
||||||
|
llm = weakref.proxy(runner.llm)
|
||||||
|
|
||||||
assert llm.get_weight_version() == "default"
|
assert llm.get_weight_version() == "default"
|
||||||
|
|
||||||
@@ -309,20 +317,19 @@ def test_full_weight_transfer_flow():
|
|||||||
|
|
||||||
|
|
||||||
@create_new_process_for_each_test()
|
@create_new_process_for_each_test()
|
||||||
def test_weight_transfer_config_backend():
|
def test_weight_transfer_config_backend(vllm_runner):
|
||||||
"""Test that WeightTransferConfig backend is properly configured."""
|
"""Test that WeightTransferConfig backend is properly configured."""
|
||||||
if torch.accelerator.device_count() < 1:
|
if torch.accelerator.device_count() < 1:
|
||||||
pytest.skip("Need at least 1 GPU for this test")
|
pytest.skip("Need at least 1 GPU for this test")
|
||||||
|
|
||||||
# Test with nccl backend
|
# Test with nccl backend
|
||||||
llm = LLM(
|
with vllm_runner(
|
||||||
model=MODEL_NAME,
|
MODEL_NAME,
|
||||||
enforce_eager=True,
|
enforce_eager=True,
|
||||||
load_format="dummy",
|
load_format="dummy",
|
||||||
tensor_parallel_size=1,
|
tensor_parallel_size=1,
|
||||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||||
)
|
) as runner:
|
||||||
|
config = runner.llm.llm_engine.vllm_config.weight_transfer_config
|
||||||
config = llm.llm_engine.vllm_config.weight_transfer_config
|
assert config is not None
|
||||||
assert config is not None
|
assert config.backend == "nccl"
|
||||||
assert config.backend == "nccl"
|
|
||||||
|
|||||||
Reference in New Issue
Block a user