[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,275 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end tests for the out-of-tree bitsandbytes plugin."""
import types
from unittest.mock import MagicMock, patch
import pytest
from transformers import BitsAndBytesConfig
from vllm_bnb_plugin import bitsandbytes_loader as bnb
from vllm.platforms import current_platform
from ...models.utils import check_embeddings_close, check_logprobs_close
from ...utils import multi_gpu_test
if current_platform.is_rocm():
from vllm.platforms.rocm import on_cdna
pytestmark = pytest.mark.skipif(
on_cdna(),
reason="bitsandbytes not supported on CDNA (warp size 64 limitation)",
)
models_4bit_to_test = [
("facebook/opt-125m", "quantize opt model inflight"),
(
"mistralai/Mistral-7B-Instruct-v0.3",
"quantize inflight model with both HF and Mistral format weights",
),
]
models_4bit_to_embedding_test = [
("intfloat/e5-mistral-7b-instruct", "quantize embedding model inflight"),
]
models_4bit_to_moe_test = [
("allenai/OLMoE-1B-7B-0125-Instruct", "quantize moe model inflight"),
]
models_pre_quant_4bit_to_test = [
(
"PrunaAI/Einstein-v6.1-Llama3-8B-bnb-4bit-smashed",
"read pre-quantized 4-bit FP4 model",
),
("poedator/opt-125m-bnb-4bit", "read pre-quantized 4-bit NF4 opt model"),
]
models_pre_quant_8bit_to_test = [
("meta-llama/Llama-Guard-3-8B-INT8", "read pre-quantized llama 8-bit model"),
("yec019/fbopt-350m-8bit", "read pre-quantized 8-bit opt model"),
]
def log_generated_texts(prompts, outputs, runner_name):
logged_texts = []
for i, (_, generated_text) in enumerate(outputs):
logged_texts.append(
{
"prompt": prompts[i],
"runner_name": runner_name,
"generated_text": generated_text,
}
)
return logged_texts
def validate_generated_texts(
hf_runner,
vllm_runner,
prompts,
model_name,
pre_quant=False,
hf_model_kwargs=None,
vllm_tp_size=1,
max_tokens=8,
):
with vllm_runner(
model_name,
quantization=None if pre_quant else "bitsandbytes",
tensor_parallel_size=vllm_tp_size,
enforce_eager=False,
default_torch_num_threads=1,
tokenizer_mode="hf",
load_format="hf",
config_format="hf",
) as llm:
vllm_outputs = llm.generate_greedy(prompts, max_tokens)
vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner")
if hf_model_kwargs is None:
hf_model_kwargs = {}
with hf_runner(
model_name, model_kwargs=hf_model_kwargs, default_torch_num_threads=1
) as llm:
hf_outputs = llm.generate_greedy(prompts, max_tokens)
hf_logs = log_generated_texts(prompts, hf_outputs, "HfRunner")
for hf_log, vllm_log in zip(hf_logs, vllm_logs):
hf_str = hf_log["generated_text"]
vllm_str = vllm_log["generated_text"]
prompt = hf_log["prompt"]
assert hf_str == vllm_str, (
f"Model: {model_name}"
f"Mismatch between HF and vLLM outputs:\n"
f"Prompt: {prompt}\n"
f"HF Output: '{hf_str}'\n"
f"vLLM Output: '{vllm_str}'"
)
@pytest.mark.parametrize("model_name, description", models_4bit_to_test)
def test_load_4bit_bnb_model(
hf_runner, vllm_runner, example_prompts, model_name, description
) -> None:
hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True))
validate_generated_texts(
hf_runner, vllm_runner, example_prompts[:1], model_name, False, hf_model_kwargs
)
@pytest.mark.parametrize("model_name, description", models_pre_quant_4bit_to_test)
def test_load_pre_quant_4bit_bnb_model(
hf_runner, vllm_runner, example_prompts, model_name, description
) -> None:
validate_generated_texts(
hf_runner, vllm_runner, example_prompts[:1], model_name, True
)
@pytest.mark.parametrize("model_name, description", models_pre_quant_8bit_to_test)
def test_load_8bit_bnb_model(
hf_runner, vllm_runner, example_prompts, model_name, description
) -> None:
validate_generated_texts(
hf_runner, vllm_runner, example_prompts[:1], model_name, True
)
@pytest.mark.parametrize("model_name, description", models_4bit_to_test)
@multi_gpu_test(num_gpus=2)
def test_load_tp_4bit_bnb_model(
hf_runner, vllm_runner, example_prompts, model_name, description
) -> None:
hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True))
validate_generated_texts(
hf_runner,
vllm_runner,
example_prompts[:1],
model_name,
False,
hf_model_kwargs,
vllm_tp_size=2,
)
@pytest.mark.parametrize("model_name, description", models_4bit_to_test)
@multi_gpu_test(num_gpus=2)
def test_load_pp_4bit_bnb_model(
hf_runner, vllm_runner, example_prompts, model_name, description
) -> None:
hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True))
validate_generated_texts(
hf_runner,
vllm_runner,
example_prompts[:1],
model_name,
False,
hf_model_kwargs,
vllm_tp_size=2,
)
@pytest.mark.skip(
reason="Need to add support for quantizing MoE experts with bnb in "
"transformers v5. See https://github.com/bitsandbytes-foundation/"
"bitsandbytes/issues/1849",
)
@pytest.mark.parametrize("model_name, description", models_4bit_to_moe_test)
def test_4bit_bnb_moe_model(
hf_runner, vllm_runner, example_prompts, model_name, description
) -> None:
hf_model_kwargs = dict(
quantization_config=BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_use_double_quant=True,
)
)
with vllm_runner(
model_name,
quantization="bitsandbytes",
enforce_eager=False,
default_torch_num_threads=1,
) as llm:
vllm_outputs = llm.generate_greedy_logprobs(
example_prompts, max_tokens=32, num_logprobs=5
)
with hf_runner(
model_name, model_kwargs=hf_model_kwargs, default_torch_num_threads=1
) as llm:
transformers_outputs = llm.generate_greedy_logprobs_limit(
example_prompts, max_tokens=32, num_logprobs=5
)
check_logprobs_close(
outputs_0_lst=transformers_outputs,
outputs_1_lst=vllm_outputs,
name_0="transformers",
name_1="vllm",
)
@pytest.mark.parametrize("model_name, description", models_4bit_to_embedding_test)
@pytest.mark.parametrize("dtype", ["half"])
def test_4bit_bnb_embedding_model(
model_name,
description,
hf_runner,
vllm_runner,
example_prompts,
dtype: str,
) -> None:
example_prompts = [str(s).strip() for s in example_prompts]
with vllm_runner(
model_name,
runner="pooling",
dtype=dtype,
gpu_memory_utilization=0.5,
quantization="bitsandbytes",
default_torch_num_threads=1,
) as vllm_model:
vllm_outputs = vllm_model.embed(example_prompts)
hf_model_kwargs = dict(quantization_config=BitsAndBytesConfig(load_in_4bit=True))
with hf_runner(
model_name,
dtype=dtype,
model_kwargs=hf_model_kwargs,
is_sentence_transformer=True,
default_torch_num_threads=1,
) as hf_model:
hf_outputs = hf_model.encode(example_prompts)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=5e-2,
)
def test_bitsandbytes_passes_revision_by_name():
fake_self = types.SimpleNamespace(
load_config=types.SimpleNamespace(download_dir="/cache"),
_get_weight_files=MagicMock(
return_value=("/folder", ["/folder/model.safetensors"], "*.safetensors")
),
)
with (
patch.object(bnb, "download_safetensors_index_file_from_hf") as mock_idx,
patch.object(
bnb,
"filter_duplicate_safetensors_files",
return_value=["/folder/model.safetensors"],
),
):
bnb.BitsAndBytesModelLoader._prepare_weights(fake_self, "org/model", "myrev")
mock_idx.assert_called_once()
assert mock_idx.call_args.kwargs.get("revision") == "myrev"
assert "myrev" not in mock_idx.call_args.args
@@ -1,70 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Transformers backend tests for bitsandbytes plugin models."""
import pytest
from vllm.platforms import current_platform
from ...models.utils import check_logprobs_close
if current_platform.is_rocm():
from vllm.platforms.rocm import on_cdna
pytestmark = pytest.mark.skipif(
on_cdna(),
reason="bitsandbytes not supported on CDNA (warp size 64 limitation)",
)
@pytest.mark.parametrize(
"model, quantization_kwargs",
[
(
"meta-llama/Llama-3.2-1B-Instruct",
{
"quantization": "bitsandbytes",
},
),
("unsloth/tinyllama-bnb-4bit", {}),
],
)
@pytest.mark.parametrize("max_tokens", [32])
@pytest.mark.parametrize("num_logprobs", [5])
def test_transformers_backend_quantization(
vllm_runner,
example_prompts,
model: str,
quantization_kwargs: dict[str, str],
max_tokens: int,
num_logprobs: int,
) -> None:
with vllm_runner(
model,
model_impl="auto",
enforce_eager=True,
**quantization_kwargs,
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs
)
with vllm_runner(
model,
model_impl="transformers",
enforce_eager=True,
**quantization_kwargs,
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
assert model_config.using_transformers_backend()
transformers_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs
)
check_logprobs_close(
outputs_0_lst=transformers_outputs,
outputs_1_lst=vllm_outputs,
name_0="transformers",
name_1="vllm",
)
@@ -1,98 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from unittest.mock import MagicMock, patch
import pytest
from vllm.config import VllmConfig
from vllm.inputs import PromptType
from vllm.outputs import PoolingRequestOutput
from vllm.plugins.io_processors import get_io_processor
from vllm.plugins.io_processors.interface import IOProcessor
from vllm.renderers import BaseRenderer
class DummyIOProcessor(IOProcessor):
"""Minimal IOProcessor used as the target of the mocked plugin entry point."""
def pre_process(
self,
prompt: object,
request_id: str | None = None,
**kwargs,
) -> PromptType | Sequence[PromptType]:
raise NotImplementedError
def post_process(
self,
model_output: Sequence[PoolingRequestOutput],
request_id: str | None = None,
**kwargs,
) -> object:
raise NotImplementedError
@pytest.fixture
def my_plugin_entry_points():
"""Patch importlib.metadata.entry_points to expose a single 'my_plugin'
entry point backed by DummyIOProcessor, exercising the full plugin-loading
code path: entry_points → plugin.load() → func() →
resolve_obj_by_qualname → IOProcessor.__init__."""
qualname = f"{DummyIOProcessor.__module__}.{DummyIOProcessor.__qualname__}"
ep = MagicMock()
ep.name = "my_plugin"
ep.value = qualname
ep.load.return_value = lambda: qualname
with patch("importlib.metadata.entry_points", return_value=[ep]):
yield
def test_loading_missing_plugin():
vllm_config = VllmConfig()
renderer = MagicMock(spec=BaseRenderer)
with pytest.raises(ValueError):
get_io_processor(
vllm_config, renderer=renderer, plugin_from_init="wrong_plugin"
)
def test_loading_plugin(my_plugin_entry_points):
# Plugin name supplied via plugin_from_init.
vllm_config = MagicMock(spec=VllmConfig)
renderer = MagicMock(spec=BaseRenderer)
result = get_io_processor(
vllm_config, renderer=renderer, plugin_from_init="my_plugin"
)
assert isinstance(result, DummyIOProcessor)
def test_loading_missing_plugin_from_model_config():
# Build a mock VllmConfig whose hf_config advertises a plugin name,
# exercising the model-config code path without loading a real model.
mock_hf_config = MagicMock()
mock_hf_config.to_dict.return_value = {"io_processor_plugin": "wrong_plugin"}
vllm_config = MagicMock(spec=VllmConfig)
vllm_config.model_config.hf_config = mock_hf_config
renderer = MagicMock(spec=BaseRenderer)
with pytest.raises(ValueError):
get_io_processor(vllm_config, renderer=renderer)
def test_loading_plugin_from_model_config(my_plugin_entry_points):
# Plugin name supplied via the model's hf_config.
mock_hf_config = MagicMock()
mock_hf_config.to_dict.return_value = {"io_processor_plugin": "my_plugin"}
vllm_config = MagicMock(spec=VllmConfig)
vllm_config.model_config.hf_config = mock_hf_config
renderer = MagicMock(spec=BaseRenderer)
result = get_io_processor(vllm_config, renderer=renderer)
assert isinstance(result, DummyIOProcessor)
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.utils import create_new_process_for_each_test
from vllm import LLM, SamplingParams
from vllm.assets.image import ImageAsset
from vllm.multimodal.image import convert_image_mode
@create_new_process_for_each_test()
def test_plugin(
monkeypatch: pytest.MonkeyPatch,
dummy_opt_path: str,
):
with monkeypatch.context() as m:
m.setenv("VLLM_PLUGINS", "")
with pytest.raises(ValueError, match="are not supported for now"):
LLM(model=dummy_opt_path, load_format="dummy")
@create_new_process_for_each_test()
def test_oot_registration_text_generation(
monkeypatch: pytest.MonkeyPatch,
dummy_opt_path: str,
):
with monkeypatch.context() as m:
m.setenv("VLLM_PLUGINS", "register_dummy_model")
prompts = ["Hello, my name is", "The text does not matter"]
sampling_params = SamplingParams(temperature=0)
llm = LLM(model=dummy_opt_path, load_format="dummy")
first_token = llm.get_tokenizer().decode(0)
outputs = llm.generate(prompts, sampling_params)
for output in outputs:
generated_text = output.outputs[0].text
# make sure only the first token is generated
rest = generated_text.replace(first_token, "")
assert rest == ""
@create_new_process_for_each_test()
def test_oot_registration_embedding(
monkeypatch: pytest.MonkeyPatch,
dummy_gemma2_embedding_path: str,
):
with monkeypatch.context() as m:
m.setenv("VLLM_PLUGINS", "register_dummy_model")
prompts = ["Hello, my name is", "The text does not matter"]
llm = LLM(
model=dummy_gemma2_embedding_path, load_format="dummy", max_model_len=512
)
outputs = llm.embed(prompts)
for output in outputs:
assert all(v == 0 for v in output.outputs.embedding)
image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
@create_new_process_for_each_test()
def test_oot_registration_multimodal(
monkeypatch: pytest.MonkeyPatch,
dummy_llava_path: str,
):
with monkeypatch.context() as m:
m.setenv("VLLM_PLUGINS", "register_dummy_model")
prompts = [
{
"prompt": "What's in the image?<image>",
"multi_modal_data": {"image": image},
},
{
"prompt": "Describe the image<image>",
"multi_modal_data": {"image": image},
},
]
sampling_params = SamplingParams(temperature=0)
llm = LLM(
model=dummy_llava_path,
load_format="dummy",
max_num_seqs=1,
trust_remote_code=True,
gpu_memory_utilization=0.98,
max_model_len=4096,
enforce_eager=True,
limit_mm_per_prompt={"image": 1},
)
first_token = llm.get_tokenizer().decode(0)
outputs = llm.generate(prompts, sampling_params)
for output in outputs:
generated_text = output.outputs[0].text
# make sure only the first token is generated
rest = generated_text.replace(first_token, "")
assert rest == ""
@@ -1,64 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import subprocess
import sys
import pytest
import torch
from vllm.plugins import load_general_plugins
def test_platform_plugins():
# simulate workload by running an example
import runpy
current_file = __file__
import os
example_file = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(current_file))),
"examples",
"basic/offline_inference/basic.py",
)
runpy.run_path(example_file)
# check if the plugin is loaded correctly
from vllm.platforms import _init_trace, current_platform
assert current_platform.device_name == "DummyDevice", (
f"Expected DummyDevice, got {current_platform.device_name}, "
"possibly because current_platform is imported before the plugin"
f" is loaded. The first import:\n{_init_trace}"
)
def test_import_vllm_does_not_resolve_platform():
# Platform plugins must be loaded after `import vllm` completes; resolving
# current_platform during it runs plugins against a half-imported vLLM.
# Needs a fresh interpreter, since vllm is already imported here.
code = (
"import sys, vllm\n"
"p = sys.modules.get('vllm.platforms')\n"
"assert p is None or p._current_platform is None, p._init_trace\n"
"from vllm.platforms import current_platform\n"
"assert current_platform.device_name == 'DummyDevice'\n"
)
subprocess.run([sys.executable, "-c", code], check=True)
def test_oot_custom_op(default_vllm_config, monkeypatch: pytest.MonkeyPatch):
# simulate workload by running an example
load_general_plugins()
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
layer = RotaryEmbedding(16, 16, 16, 16, True, torch.float16)
assert layer.__class__.__name__ == "DummyRotaryEmbedding", (
f"Expected DummyRotaryEmbedding, got {layer.__class__.__name__}, "
"possibly because the custom op is not registered correctly."
)
assert hasattr(layer, "addition_config"), (
"Expected DummyRotaryEmbedding to have an 'addition_config' attribute, "
"which is set by the custom op."
)