[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,275 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end tests for the out-of-tree bitsandbytes plugin."""
|
||||
|
||||
import types
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from transformers import BitsAndBytesConfig
|
||||
from vllm_bnb_plugin import bitsandbytes_loader as bnb
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...models.utils import check_embeddings_close, check_logprobs_close
|
||||
from ...utils import multi_gpu_test
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_cdna
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
on_cdna(),
|
||||
reason="bitsandbytes not supported on CDNA (warp size 64 limitation)",
|
||||
)
|
||||
|
||||
models_4bit_to_test = [
|
||||
("facebook/opt-125m", "quantize opt model inflight"),
|
||||
(
|
||||
"mistralai/Mistral-7B-Instruct-v0.3",
|
||||
"quantize inflight model with both HF and Mistral format weights",
|
||||
),
|
||||
]
|
||||
|
||||
models_4bit_to_embedding_test = [
|
||||
("intfloat/e5-mistral-7b-instruct", "quantize embedding model inflight"),
|
||||
]
|
||||
|
||||
models_4bit_to_moe_test = [
|
||||
("allenai/OLMoE-1B-7B-0125-Instruct", "quantize moe model inflight"),
|
||||
]
|
||||
|
||||
models_pre_quant_4bit_to_test = [
|
||||
(
|
||||
"PrunaAI/Einstein-v6.1-Llama3-8B-bnb-4bit-smashed",
|
||||
"read pre-quantized 4-bit FP4 model",
|
||||
),
|
||||
("poedator/opt-125m-bnb-4bit", "read pre-quantized 4-bit NF4 opt model"),
|
||||
]
|
||||
|
||||
models_pre_quant_8bit_to_test = [
|
||||
("meta-llama/Llama-Guard-3-8B-INT8", "read pre-quantized llama 8-bit model"),
|
||||
("yec019/fbopt-350m-8bit", "read pre-quantized 8-bit opt model"),
|
||||
]
|
||||
|
||||
|
||||
def log_generated_texts(prompts, outputs, runner_name):
|
||||
logged_texts = []
|
||||
for i, (_, generated_text) in enumerate(outputs):
|
||||
logged_texts.append(
|
||||
{
|
||||
"prompt": prompts[i],
|
||||
"runner_name": runner_name,
|
||||
"generated_text": generated_text,
|
||||
}
|
||||
)
|
||||
return logged_texts
|
||||
|
||||
|
||||
def validate_generated_texts(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
prompts,
|
||||
model_name,
|
||||
pre_quant=False,
|
||||
hf_model_kwargs=None,
|
||||
vllm_tp_size=1,
|
||||
max_tokens=8,
|
||||
):
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
quantization=None if pre_quant else "bitsandbytes",
|
||||
tensor_parallel_size=vllm_tp_size,
|
||||
enforce_eager=False,
|
||||
default_torch_num_threads=1,
|
||||
tokenizer_mode="hf",
|
||||
load_format="hf",
|
||||
config_format="hf",
|
||||
) as llm:
|
||||
vllm_outputs = llm.generate_greedy(prompts, max_tokens)
|
||||
vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner")
|
||||
|
||||
if hf_model_kwargs is None:
|
||||
hf_model_kwargs = {}
|
||||
|
||||
with hf_runner(
|
||||
model_name, model_kwargs=hf_model_kwargs, default_torch_num_threads=1
|
||||
) as llm:
|
||||
hf_outputs = llm.generate_greedy(prompts, max_tokens)
|
||||
hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner")
|
||||
|
||||
for hf_log, vllm_log in zip(hf_logs, vllm_logs):
|
||||
hf_str = hf_log["generated_text"]
|
||||
vllm_str = vllm_log["generated_text"]
|
||||
prompt = hf_log["prompt"]
|
||||
assert hf_str == vllm_str, (
|
||||
f"Model: {model_name}"
|
||||
f"Mismatch between HF and vLLM outputs:\n"
|
||||
f"Prompt: {prompt}\n"
|
||||
f"HF Output: '{hf_str}'\n"
|
||||
f"vLLM Output: '{vllm_str}'"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name, description", models_4bit_to_test)
|
||||
def test_load_4bit_bnb_model(
|
||||
hf_runner, vllm_runner, example_prompts, model_name, description
|
||||
) -> None:
|
||||
hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True))
|
||||
validate_generated_texts(
|
||||
hf_runner, vllm_runner, example_prompts[:1], model_name, False, hf_model_kwargs
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name, description", models_pre_quant_4bit_to_test)
|
||||
def test_load_pre_quant_4bit_bnb_model(
|
||||
hf_runner, vllm_runner, example_prompts, model_name, description
|
||||
) -> None:
|
||||
validate_generated_texts(
|
||||
hf_runner, vllm_runner, example_prompts[:1], model_name, True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name, description", models_pre_quant_8bit_to_test)
|
||||
def test_load_8bit_bnb_model(
|
||||
hf_runner, vllm_runner, example_prompts, model_name, description
|
||||
) -> None:
|
||||
validate_generated_texts(
|
||||
hf_runner, vllm_runner, example_prompts[:1], model_name, True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name, description", models_4bit_to_test)
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_load_tp_4bit_bnb_model(
|
||||
hf_runner, vllm_runner, example_prompts, model_name, description
|
||||
) -> None:
|
||||
hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True))
|
||||
validate_generated_texts(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts[:1],
|
||||
model_name,
|
||||
False,
|
||||
hf_model_kwargs,
|
||||
vllm_tp_size=2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name, description", models_4bit_to_test)
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_load_pp_4bit_bnb_model(
|
||||
hf_runner, vllm_runner, example_prompts, model_name, description
|
||||
) -> None:
|
||||
hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True))
|
||||
validate_generated_texts(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts[:1],
|
||||
model_name,
|
||||
False,
|
||||
hf_model_kwargs,
|
||||
vllm_tp_size=2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Need to add support for quantizing MoE experts with bnb in "
|
||||
"transformers v5. See https://github.com/bitsandbytes-foundation/"
|
||||
"bitsandbytes/issues/1849",
|
||||
)
|
||||
@pytest.mark.parametrize("model_name, description", models_4bit_to_moe_test)
|
||||
def test_4bit_bnb_moe_model(
|
||||
hf_runner, vllm_runner, example_prompts, model_name, description
|
||||
) -> None:
|
||||
hf_model_kwargs = dict(
|
||||
quantization_config=BitsAndBytesConfig(
|
||||
load_in_4bit=True,
|
||||
bnb_4bit_quant_type="nf4",
|
||||
bnb_4bit_use_double_quant=True,
|
||||
)
|
||||
)
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
quantization="bitsandbytes",
|
||||
enforce_eager=False,
|
||||
default_torch_num_threads=1,
|
||||
) as llm:
|
||||
vllm_outputs = llm.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
|
||||
with hf_runner(
|
||||
model_name, model_kwargs=hf_model_kwargs, default_torch_num_threads=1
|
||||
) as llm:
|
||||
transformers_outputs = llm.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=transformers_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="transformers",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name, description", models_4bit_to_embedding_test)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_4bit_bnb_embedding_model(
|
||||
model_name,
|
||||
description,
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
example_prompts = [str(s).strip() for s in example_prompts]
|
||||
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
runner="pooling",
|
||||
dtype=dtype,
|
||||
gpu_memory_utilization=0.5,
|
||||
quantization="bitsandbytes",
|
||||
default_torch_num_threads=1,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.embed(example_prompts)
|
||||
|
||||
hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True))
|
||||
with hf_runner(
|
||||
model_name,
|
||||
dtype=dtype,
|
||||
model_kwargs=hf_model_kwargs,
|
||||
is_sentence_transformer=True,
|
||||
default_torch_num_threads=1,
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.encode(example_prompts)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=5e-2,
|
||||
)
|
||||
|
||||
|
||||
def test_bitsandbytes_passes_revision_by_name():
|
||||
fake_self = types.SimpleNamespace(
|
||||
load_config=types.SimpleNamespace(download_dir="/cache"),
|
||||
_get_weight_files=MagicMock(
|
||||
return_value=("/folder", ["/folder/model.safetensors"], "*.safetensors")
|
||||
),
|
||||
)
|
||||
with (
|
||||
patch.object(bnb, "download_safetensors_index_file_from_hf") as mock_idx,
|
||||
patch.object(
|
||||
bnb,
|
||||
"filter_duplicate_safetensors_files",
|
||||
return_value=["/folder/model.safetensors"],
|
||||
),
|
||||
):
|
||||
bnb.BitsAndBytesModelLoader._prepare_weights(fake_self, "org/model", "myrev")
|
||||
|
||||
mock_idx.assert_called_once()
|
||||
assert mock_idx.call_args.kwargs.get("revision") == "myrev"
|
||||
assert "myrev" not in mock_idx.call_args.args
|
||||
@@ -1,70 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Transformers backend tests for bitsandbytes plugin models."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...models.utils import check_logprobs_close
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_cdna
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
on_cdna(),
|
||||
reason="bitsandbytes not supported on CDNA (warp size 64 limitation)",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, quantization_kwargs",
|
||||
[
|
||||
(
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
{
|
||||
"quantization": "bitsandbytes",
|
||||
},
|
||||
),
|
||||
("unsloth/tinyllama-bnb-4bit", {}),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_transformers_backend_quantization(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
quantization_kwargs: dict[str, str],
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
model_impl="auto",
|
||||
enforce_eager=True,
|
||||
**quantization_kwargs,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
model_impl="transformers",
|
||||
enforce_eager=True,
|
||||
**quantization_kwargs,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
assert model_config.using_transformers_backend()
|
||||
|
||||
transformers_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=transformers_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="transformers",
|
||||
name_1="vllm",
|
||||
)
|
||||
@@ -1,98 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.plugins.io_processors import get_io_processor
|
||||
from vllm.plugins.io_processors.interface import IOProcessor
|
||||
from vllm.renderers import BaseRenderer
|
||||
|
||||
|
||||
class DummyIOProcessor(IOProcessor):
|
||||
"""Minimal IOProcessor used as the target of the mocked plugin entry point."""
|
||||
|
||||
def pre_process(
|
||||
self,
|
||||
prompt: object,
|
||||
request_id: str | None = None,
|
||||
**kwargs,
|
||||
) -> PromptType | Sequence[PromptType]:
|
||||
raise NotImplementedError
|
||||
|
||||
def post_process(
|
||||
self,
|
||||
model_output: Sequence[PoolingRequestOutput],
|
||||
request_id: str | None = None,
|
||||
**kwargs,
|
||||
) -> object:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def my_plugin_entry_points():
|
||||
"""Patch importlib.metadata.entry_points to expose a single 'my_plugin'
|
||||
entry point backed by DummyIOProcessor, exercising the full plugin-loading
|
||||
code path: entry_points → plugin.load() → func() →
|
||||
resolve_obj_by_qualname → IOProcessor.__init__."""
|
||||
qualname = f"{DummyIOProcessor.__module__}.{DummyIOProcessor.__qualname__}"
|
||||
ep = MagicMock()
|
||||
ep.name = "my_plugin"
|
||||
ep.value = qualname
|
||||
ep.load.return_value = lambda: qualname
|
||||
with patch("importlib.metadata.entry_points", return_value=[ep]):
|
||||
yield
|
||||
|
||||
|
||||
def test_loading_missing_plugin():
|
||||
vllm_config = VllmConfig()
|
||||
renderer = MagicMock(spec=BaseRenderer)
|
||||
with pytest.raises(ValueError):
|
||||
get_io_processor(
|
||||
vllm_config, renderer=renderer, plugin_from_init="wrong_plugin"
|
||||
)
|
||||
|
||||
|
||||
def test_loading_plugin(my_plugin_entry_points):
|
||||
# Plugin name supplied via plugin_from_init.
|
||||
vllm_config = MagicMock(spec=VllmConfig)
|
||||
renderer = MagicMock(spec=BaseRenderer)
|
||||
|
||||
result = get_io_processor(
|
||||
vllm_config, renderer=renderer, plugin_from_init="my_plugin"
|
||||
)
|
||||
|
||||
assert isinstance(result, DummyIOProcessor)
|
||||
|
||||
|
||||
def test_loading_missing_plugin_from_model_config():
|
||||
# Build a mock VllmConfig whose hf_config advertises a plugin name,
|
||||
# exercising the model-config code path without loading a real model.
|
||||
mock_hf_config = MagicMock()
|
||||
mock_hf_config.to_dict.return_value = {"io_processor_plugin": "wrong_plugin"}
|
||||
|
||||
vllm_config = MagicMock(spec=VllmConfig)
|
||||
vllm_config.model_config.hf_config = mock_hf_config
|
||||
|
||||
renderer = MagicMock(spec=BaseRenderer)
|
||||
with pytest.raises(ValueError):
|
||||
get_io_processor(vllm_config, renderer=renderer)
|
||||
|
||||
|
||||
def test_loading_plugin_from_model_config(my_plugin_entry_points):
|
||||
# Plugin name supplied via the model's hf_config.
|
||||
mock_hf_config = MagicMock()
|
||||
mock_hf_config.to_dict.return_value = {"io_processor_plugin": "my_plugin"}
|
||||
|
||||
vllm_config = MagicMock(spec=VllmConfig)
|
||||
vllm_config.model_config.hf_config = mock_hf_config
|
||||
|
||||
renderer = MagicMock(spec=BaseRenderer)
|
||||
|
||||
result = get_io_processor(vllm_config, renderer=renderer)
|
||||
|
||||
assert isinstance(result, DummyIOProcessor)
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_plugin(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dummy_opt_path: str,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_PLUGINS", "")
|
||||
|
||||
with pytest.raises(ValueError, match="are not supported for now"):
|
||||
LLM(model=dummy_opt_path, load_format="dummy")
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_oot_registration_text_generation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dummy_opt_path: str,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_PLUGINS", "register_dummy_model")
|
||||
prompts = ["Hello, my name is", "The text does not matter"]
|
||||
sampling_params = SamplingParams(temperature=0)
|
||||
llm = LLM(model=dummy_opt_path, load_format="dummy")
|
||||
first_token = llm.get_tokenizer().decode(0)
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
|
||||
for output in outputs:
|
||||
generated_text = output.outputs[0].text
|
||||
# make sure only the first token is generated
|
||||
rest = generated_text.replace(first_token, "")
|
||||
assert rest == ""
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_oot_registration_embedding(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dummy_gemma2_embedding_path: str,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_PLUGINS", "register_dummy_model")
|
||||
prompts = ["Hello, my name is", "The text does not matter"]
|
||||
llm = LLM(
|
||||
model=dummy_gemma2_embedding_path, load_format="dummy", max_model_len=512
|
||||
)
|
||||
outputs = llm.embed(prompts)
|
||||
|
||||
for output in outputs:
|
||||
assert all(v == 0 for v in output.outputs.embedding)
|
||||
|
||||
|
||||
image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_oot_registration_multimodal(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dummy_llava_path: str,
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_PLUGINS", "register_dummy_model")
|
||||
prompts = [
|
||||
{
|
||||
"prompt": "What's in the image?<image>",
|
||||
"multi_modal_data": {"image": image},
|
||||
},
|
||||
{
|
||||
"prompt": "Describe the image<image>",
|
||||
"multi_modal_data": {"image": image},
|
||||
},
|
||||
]
|
||||
|
||||
sampling_params = SamplingParams(temperature=0)
|
||||
llm = LLM(
|
||||
model=dummy_llava_path,
|
||||
load_format="dummy",
|
||||
max_num_seqs=1,
|
||||
trust_remote_code=True,
|
||||
gpu_memory_utilization=0.98,
|
||||
max_model_len=4096,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
|
||||
first_token = llm.get_tokenizer().decode(0)
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
|
||||
for output in outputs:
|
||||
generated_text = output.outputs[0].text
|
||||
# make sure only the first token is generated
|
||||
rest = generated_text.replace(first_token, "")
|
||||
assert rest == ""
|
||||
@@ -1,64 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.plugins import load_general_plugins
|
||||
|
||||
|
||||
def test_platform_plugins():
|
||||
# simulate workload by running an example
|
||||
import runpy
|
||||
|
||||
current_file = __file__
|
||||
import os
|
||||
|
||||
example_file = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(current_file))),
|
||||
"examples",
|
||||
"basic/offline_inference/basic.py",
|
||||
)
|
||||
runpy.run_path(example_file)
|
||||
|
||||
# check if the plugin is loaded correctly
|
||||
from vllm.platforms import _init_trace, current_platform
|
||||
|
||||
assert current_platform.device_name == "DummyDevice", (
|
||||
f"Expected DummyDevice, got {current_platform.device_name}, "
|
||||
"possibly because current_platform is imported before the plugin"
|
||||
f" is loaded. The first import:\n{_init_trace}"
|
||||
)
|
||||
|
||||
|
||||
def test_import_vllm_does_not_resolve_platform():
|
||||
# Platform plugins must be loaded after `import vllm` completes; resolving
|
||||
# current_platform during it runs plugins against a half-imported vLLM.
|
||||
# Needs a fresh interpreter, since vllm is already imported here.
|
||||
code = (
|
||||
"import sys, vllm\n"
|
||||
"p = sys.modules.get('vllm.platforms')\n"
|
||||
"assert p is None or p._current_platform is None, p._init_trace\n"
|
||||
"from vllm.platforms import current_platform\n"
|
||||
"assert current_platform.device_name == 'DummyDevice'\n"
|
||||
)
|
||||
subprocess.run([sys.executable, "-c", code], check=True)
|
||||
|
||||
|
||||
def test_oot_custom_op(default_vllm_config, monkeypatch: pytest.MonkeyPatch):
|
||||
# simulate workload by running an example
|
||||
load_general_plugins()
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
|
||||
layer = RotaryEmbedding(16, 16, 16, 16, True, torch.float16)
|
||||
assert layer.__class__.__name__ == "DummyRotaryEmbedding", (
|
||||
f"Expected DummyRotaryEmbedding, got {layer.__class__.__name__}, "
|
||||
"possibly because the custom op is not registered correctly."
|
||||
)
|
||||
assert hasattr(layer, "addition_config"), (
|
||||
"Expected DummyRotaryEmbedding to have an 'addition_config' attribute, "
|
||||
"which is set by the custom op."
|
||||
)
|
||||
Reference in New Issue
Block a user