[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
@@ -1,236 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
import requests
# Test configuration for BGE-M3 sparse plugin
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.pooling.protocol import IOProcessorResponse
model_config = {
"model_name": "BAAI/bge-m3",
"plugin": "bge_m3_sparse_plugin",
"test_input": "What is the capital of France?",
"hf_overrides": json.dumps(
{"architectures": ["BgeM3EmbeddingModel"], "head_dtype": "float16"}
),
}
dense_embedding_sum = [
-0.7214539647102356, # "What is the capital of France?"
-0.6926871538162231, # "What is the capital of Germany?"
-0.7129564881324768, # "What is the capital of Spain?"
]
def _float_close(expected: object, result: object):
assert isinstance(expected, float) and isinstance(result, float), (
f"{expected=} or {result=} is not float"
)
return (expected - result) < 1e-3 or abs(expected / result - 1) < 1e-3
def _get_attr_or_val(obj: object | dict, key: str):
if isinstance(obj, dict) and key in obj:
return obj[key]
return getattr(obj, key, None)
def _check_dense_embedding(data, index=0):
assert _float_close(sum(data), dense_embedding_sum[index]), (
"dense-embedding result not match"
)
def _check_sparse_embedding(data, check_tokens=False):
expected_weights = [
{"token_id": 32, "weight": 0.0552978515625, "token": "?"},
{"token_id": 70, "weight": 0.09808349609375, "token": " the"},
{"token_id": 83, "weight": 0.08154296875, "token": " is"},
{"token_id": 111, "weight": 0.11810302734375, "token": " of"},
{"token_id": 4865, "weight": 0.1171875, "token": " What"},
{"token_id": 9942, "weight": 0.292236328125, "token": " France"},
{"token_id": 10323, "weight": 0.2802734375, "token": " capital"},
]
expected_embed = {x["token_id"]: x for x in expected_weights}
assert len(data) == len(expected_embed)
for entry in data:
expected_val = expected_embed[_get_attr_or_val(entry, "token_id")]
assert _float_close(
expected_val["weight"], _get_attr_or_val(entry, "weight")
), f"actual embed {entry} not equal to {expected_val}"
if check_tokens:
assert expected_val["token"] == _get_attr_or_val(entry, "token"), (
f"actual embed {entry} not equal to {expected_val}"
)
else:
assert _get_attr_or_val(entry, "token") is None, (
f"{entry} should not return token"
)
@pytest.fixture(scope="function")
def server():
args = [
"--runner",
"pooling",
"--enforce-eager",
"--max-num-seqs",
"32",
"--hf_overrides",
model_config["hf_overrides"],
"--io-processor-plugin",
model_config["plugin"],
]
with RemoteOpenAIServer(model_config["model_name"], args) as remote_server:
yield remote_server
@pytest.mark.asyncio
@pytest.mark.parametrize(
"return_tokens",
[True, False],
)
async def test_bge_m3_sparse_plugin_online(
server: RemoteOpenAIServer, return_tokens: bool
):
"""Test BGE-M3 sparse plugin in online mode via API."""
request_payload = {
"model": model_config["model_name"],
"task": "plugin",
"data": {"input": model_config["test_input"], "return_tokens": return_tokens},
}
ret = requests.post(
server.url_for("pooling"),
json=request_payload,
)
response = ret.json()
# Verify the request response is in the correct format
parsed_response: object = IOProcessorResponse(**response).data
assert parsed_response
# Verify the output is formatted as expected for this plugin
assert _get_attr_or_val(parsed_response, "data")
assert len(_get_attr_or_val(parsed_response, "data")) > 0
data_entry = _get_attr_or_val(parsed_response, "data")[0]
assert _get_attr_or_val(data_entry, "object") == "dense&sparse"
assert _get_attr_or_val(data_entry, "sparse_embedding")
# Verify sparse embedding format
sparse_embedding = _get_attr_or_val(data_entry, "sparse_embedding")
assert isinstance(sparse_embedding, list)
_check_sparse_embedding(sparse_embedding, return_tokens)
# Verify dense embedding format
dense_embedding = _get_attr_or_val(data_entry, "dense_embedding")
assert isinstance(dense_embedding, list)
_check_dense_embedding(dense_embedding)
# Verify usage information
usage = _get_attr_or_val(parsed_response, "usage")
assert usage, f"usage not found for {parsed_response}"
assert _get_attr_or_val(usage, "prompt_tokens") > 0
assert _get_attr_or_val(usage, "total_tokens") == _get_attr_or_val(
usage, "prompt_tokens"
)
@pytest.mark.parametrize(
"return_tokens",
[True, False],
)
def test_bge_m3_sparse_plugin_offline(vllm_runner, return_tokens: bool):
"""Test BGE-M3 sparse plugin in offline mode."""
prompt = {
"data": {
"input": model_config["test_input"],
"return_tokens": return_tokens,
}
}
with vllm_runner(
model_config["model_name"],
runner="pooling",
enforce_eager=True,
max_num_seqs=32,
io_processor_plugin=model_config["plugin"],
hf_overrides=json.loads(model_config["hf_overrides"]),
default_torch_num_threads=1,
) as llm_runner:
llm = llm_runner.get_llm()
pooler_output = llm.encode(prompt, pooling_task="plugin")
outputs = pooler_output[0]
# Verify output structure
assert hasattr(outputs, "outputs")
response = outputs.outputs
assert hasattr(response, "data")
assert len(response.data) == 1
# Verify response data
for i, output in enumerate(response.data):
# Each output should have sparse embeddings
sparse_embedding = output.sparse_embedding
assert isinstance(sparse_embedding, list)
_check_sparse_embedding(sparse_embedding, return_tokens)
dense_embedding = output.dense_embedding
assert isinstance(dense_embedding, list)
_check_dense_embedding(dense_embedding)
# Verify usage
assert response.usage.prompt_tokens > 0
assert response.usage.total_tokens == response.usage.prompt_tokens
def test_bge_m3_sparse_plugin_offline_multiple_inputs(vllm_runner):
"""Test BGE-M3 sparse plugin with multiple inputs in offline mode."""
prompts = {
"data": {
"input": [
"What is the capital of France?",
"What is the capital of Germany?",
"What is the capital of Spain?",
],
"return_tokens": True,
}
}
with vllm_runner(
model_config["model_name"],
runner="pooling",
enforce_eager=True,
max_num_seqs=32,
io_processor_plugin=model_config["plugin"],
hf_overrides=json.loads(model_config["hf_overrides"]),
default_torch_num_threads=1,
) as llm_runner:
llm = llm_runner.get_llm()
pooler_output = llm.encode(prompts, pooling_task="plugin")
outputs = pooler_output[0]
# Verify output structure
assert hasattr(outputs, "outputs")
response = outputs.outputs
assert hasattr(response, "data")
assert len(response.data) == 3
for i, output in enumerate(response.data):
# Each output should have sparse embeddings
sparse_embedding = output.sparse_embedding
assert isinstance(sparse_embedding, list)
dense_embedding = output.dense_embedding
assert isinstance(dense_embedding, list)
_check_dense_embedding(dense_embedding, i)
# Verify usage
assert response.usage.prompt_tokens > 0
assert response.usage.total_tokens == response.usage.prompt_tokens
@@ -1,222 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from typing import TypedDict
import pytest
import requests
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.pooling.protocol import IOProcessorResponse
# Test configuration for ColBERT query plugin
class ModelConfig(TypedDict):
model_name: str
plugin: str
query_input: str
document_input: str
hf_overrides: str
embedding_dim: int
query_maxlen: int
model_config: ModelConfig = {
"model_name": "jinaai/jina-colbert-v2",
"plugin": "colbert_query_plugin",
"query_input": "What is machine learning?",
"document_input": "Machine learning is a subset of artificial intelligence.",
"hf_overrides": json.dumps({"architectures": ["ColBERTJinaRobertaModel"]}),
"embedding_dim": 128,
"query_maxlen": 32,
}
def _get_attr_or_val(obj: object | dict, key: str):
if isinstance(obj, dict) and key in obj:
return obj[key]
return getattr(obj, key, None)
def _check_token_embeddings(entry, expected_input_type: str):
assert _get_attr_or_val(entry, "object") == "embedding"
assert _get_attr_or_val(entry, "input_type") == expected_input_type
embedding = _get_attr_or_val(entry, "embedding")
assert isinstance(embedding, list) and len(embedding) > 0
for token_embedding in embedding:
assert isinstance(token_embedding, list)
assert len(token_embedding) == model_config["embedding_dim"]
return embedding
@pytest.fixture(scope="module")
def server():
args = [
"--runner",
"pooling",
"--enforce-eager",
"--max-num-seqs",
"32",
"--trust-remote-code",
"--hf_overrides",
model_config["hf_overrides"],
"--io-processor-plugin",
model_config["plugin"],
]
with RemoteOpenAIServer(model_config["model_name"], args) as remote_server:
yield remote_server
def _post_pooling(server: RemoteOpenAIServer, data: dict):
request_payload = {
"model": model_config["model_name"],
"task": "plugin",
"data": data,
}
ret = requests.post(server.url_for("pooling"), json=request_payload)
ret.raise_for_status()
response = ret.json()
parsed_response: object = IOProcessorResponse(**response).data
assert parsed_response
return parsed_response
def test_colbert_query_plugin_query_online(server: RemoteOpenAIServer):
"""Queries are expanded to exactly query_maxlen token vectors."""
parsed_response = _post_pooling(
server, {"input": model_config["query_input"], "input_type": "query"}
)
data = _get_attr_or_val(parsed_response, "data")
assert len(data) == 1
embedding = _check_token_embeddings(data[0], "query")
assert len(embedding) == model_config["query_maxlen"]
usage = _get_attr_or_val(parsed_response, "usage")
assert _get_attr_or_val(usage, "prompt_tokens") == model_config["query_maxlen"]
def test_colbert_query_plugin_document_online(server: RemoteOpenAIServer):
"""Documents return one vector per token, with no mask expansion."""
parsed_response = _post_pooling(
server, {"input": model_config["document_input"], "input_type": "document"}
)
data = _get_attr_or_val(parsed_response, "data")
assert len(data) == 1
embedding = _check_token_embeddings(data[0], "document")
# No query expansion: number of vectors tracks the input length.
assert len(embedding) != model_config["query_maxlen"]
usage = _get_attr_or_val(parsed_response, "usage")
assert _get_attr_or_val(usage, "prompt_tokens") == len(embedding)
def test_colbert_query_plugin_missing_input_type_online(server: RemoteOpenAIServer):
"""input_type is required; omitting it is rejected."""
request_payload = {
"model": model_config["model_name"],
"task": "plugin",
"data": {"input": model_config["document_input"]},
}
ret = requests.post(server.url_for("pooling"), json=request_payload)
assert ret.status_code == 400
def test_colbert_query_plugin_batch_online(server: RemoteOpenAIServer):
"""A list input returns one entry per prompt."""
queries = ["What is machine learning?", "What is deep learning?"]
parsed_response = _post_pooling(server, {"input": queries, "input_type": "query"})
data = _get_attr_or_val(parsed_response, "data")
assert len(data) == len(queries)
for i, entry in enumerate(data):
assert _get_attr_or_val(entry, "index") == i
embedding = _check_token_embeddings(entry, "query")
assert len(embedding) == model_config["query_maxlen"]
@pytest.mark.parametrize("input_type", ["query", "document"])
def test_colbert_query_plugin_offline(vllm_runner, input_type: str):
"""Test the ColBERT query plugin in offline mode."""
input_text = (
model_config["query_input"]
if input_type == "query"
else model_config["document_input"]
)
prompt = {
"data": {
"input": input_text,
"input_type": input_type,
}
}
with vllm_runner(
model_config["model_name"],
runner="pooling",
enforce_eager=True,
max_num_seqs=32,
trust_remote_code=True,
io_processor_plugin=model_config["plugin"],
hf_overrides=json.loads(model_config["hf_overrides"]),
default_torch_num_threads=1,
) as llm_runner:
llm = llm_runner.get_llm()
pooler_output = llm.encode(prompt, pooling_task="plugin")
response = pooler_output[0].outputs
assert len(response.data) == 1
embedding = _check_token_embeddings(response.data[0], input_type)
if input_type == "query":
assert len(embedding) == model_config["query_maxlen"]
else:
assert len(embedding) != model_config["query_maxlen"]
assert response.usage.prompt_tokens == len(embedding)
assert response.usage.total_tokens == response.usage.prompt_tokens
def test_colbert_query_plugin_offline_multiple_inputs(vllm_runner):
"""Test the ColBERT query plugin with multiple inputs in offline mode."""
queries = [
"What is machine learning?",
"What is deep learning?",
"Why?",
]
prompts = {
"data": {
"input": queries,
"input_type": "query",
}
}
with vllm_runner(
model_config["model_name"],
runner="pooling",
enforce_eager=True,
max_num_seqs=32,
trust_remote_code=True,
io_processor_plugin=model_config["plugin"],
hf_overrides=json.loads(model_config["hf_overrides"]),
default_torch_num_threads=1,
) as llm_runner:
llm = llm_runner.get_llm()
pooler_output = llm.encode(prompts, pooling_task="plugin")
response = pooler_output[0].outputs
assert len(response.data) == len(queries)
for i, entry in enumerate(response.data):
assert entry.index == i
embedding = _check_token_embeddings(entry, "query")
assert len(embedding) == model_config["query_maxlen"]
expected_tokens = model_config["query_maxlen"] * len(queries)
assert response.usage.prompt_tokens == expected_tokens
assert response.usage.total_tokens == response.usage.prompt_tokens
@@ -1,154 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
import io
import imagehash
import pybase64 as base64
import pytest
import requests
from PIL import Image
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.pooling.protocol import IOProcessorResponse
pytestmark = pytest.mark.skipif(
importlib.util.find_spec("terratorch") is None,
reason="terratorch unavailable while PyPI has `lightning` quarantined; see #41376",
)
models_config = {
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11": {
"image_url": "https://huggingface.co/christian-pinto/Prithvi-EO-2.0-300M-TL-VLLM/resolve/main/valencia_example_2024-10-26.tiff", # noqa: E501
"out_hash": "aa6d92ad25926a5e",
"plugin": "prithvi_to_tiff",
},
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-BurnScars": {
"image_url": "https://huggingface.co/ibm-nasa-geospatial/Prithvi-EO-2.0-300M-BurnScars/resolve/main/examples/subsetted_512x512_HLS.S30.T10SEH.2018190.v1.4_merged.tif", # noqa: E501
"out_hash": "c07f4f602da73552",
"plugin": "prithvi_to_tiff",
},
}
def _compute_image_hash(base64_data: str) -> str:
# Decode the base64 output and create image from byte stream
decoded_image = base64.b64decode(base64_data)
image = Image.open(io.BytesIO(decoded_image))
# Compute perceptual hash of the output image
return str(imagehash.phash(image))
@pytest.fixture(scope="function")
def server(model_name, plugin):
args = [
"--runner",
"pooling",
"--enforce-eager",
"--skip-tokenizer-init",
# Limit the maximum number of parallel requests
# to avoid the model going OOM in CI.
"--max-num-seqs",
"32",
"--io-processor-plugin",
plugin,
"--enable-mm-embeds",
]
with RemoteOpenAIServer(model_name, args) as remote_server:
yield remote_server
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_name, image_url, plugin, expected_hash",
[
(model_name, config["image_url"], config["plugin"], config["out_hash"])
for model_name, config in models_config.items()
],
)
async def test_prithvi_mae_plugin_online(
server: RemoteOpenAIServer,
model_name: str,
image_url: str | dict,
plugin: str,
expected_hash: str,
):
request_payload_url = {
"data": {
"data": image_url,
"data_format": "url",
"image_format": "tiff",
"out_data_format": "b64_json",
},
"priority": 0,
"model": model_name,
"softmax": False,
}
ret = requests.post(
server.url_for("pooling"),
json=request_payload_url,
)
response = ret.json()
# verify the request response is in the correct format
parsed_response: IOProcessorResponse = IOProcessorResponse(**response)
assert parsed_response
# verify the output is formatted as expected for this plugin
plugin_data = parsed_response.data
assert all(plugin_data.get(attr) for attr in ["type", "format", "data"])
# Compute the output image hash and compare it against the expected hash
image_hash = _compute_image_hash(plugin_data["data"])
assert image_hash == expected_hash, (
f"Image hash mismatch: expected {expected_hash}, got {image_hash}"
)
@pytest.mark.parametrize(
"model_name, image_url, plugin, expected_hash",
[
(model_name, config["image_url"], config["plugin"], config["out_hash"])
for model_name, config in models_config.items()
],
)
def test_prithvi_mae_plugin_offline(
vllm_runner, model_name: str, image_url: str | dict, plugin: str, expected_hash: str
):
img_data = dict(
data=image_url,
data_format="url",
image_format="tiff",
out_data_format="b64_json",
)
prompt = dict(data=img_data)
with vllm_runner(
model_name,
runner="pooling",
skip_tokenizer_init=True,
enable_mm_embeds=True,
enforce_eager=True,
# Limit the maximum number of parallel requests
# to avoid the model going OOM in CI.
max_num_seqs=32,
io_processor_plugin=plugin,
default_torch_num_threads=1,
) as llm_runner:
pooler_output = llm_runner.get_llm().encode(prompt, pooling_task="plugin")
output = pooler_output[0].outputs
# verify the output is formatted as expected for this plugin
assert all(hasattr(output, attr) for attr in ["type", "format", "data"])
# Compute the output image hash and compare it against the expected hash
image_hash = _compute_image_hash(output.data)
assert image_hash == expected_hash, (
f"Image hash mismatch: expected {expected_hash}, got {image_hash}"
)