[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,236 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
# Test configuration for BGE-M3 sparse plugin
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.pooling.protocol import IOProcessorResponse
|
||||
|
||||
model_config = {
|
||||
"model_name": "BAAI/bge-m3",
|
||||
"plugin": "bge_m3_sparse_plugin",
|
||||
"test_input": "What is the capital of France?",
|
||||
"hf_overrides": json.dumps(
|
||||
{"architectures": ["BgeM3EmbeddingModel"], "head_dtype": "float16"}
|
||||
),
|
||||
}
|
||||
|
||||
dense_embedding_sum = [
|
||||
-0.7214539647102356, # "What is the capital of France?"
|
||||
-0.6926871538162231, # "What is the capital of Germany?"
|
||||
-0.7129564881324768, # "What is the capital of Spain?"
|
||||
]
|
||||
|
||||
|
||||
def _float_close(expected: object, result: object):
|
||||
assert isinstance(expected, float) and isinstance(result, float), (
|
||||
f"{expected=} or {result=} is not float"
|
||||
)
|
||||
return (expected - result) < 1e-3 or abs(expected / result - 1) < 1e-3
|
||||
|
||||
|
||||
def _get_attr_or_val(obj: object | dict, key: str):
|
||||
if isinstance(obj, dict) and key in obj:
|
||||
return obj[key]
|
||||
return getattr(obj, key, None)
|
||||
|
||||
|
||||
def _check_dense_embedding(data, index=0):
|
||||
assert _float_close(sum(data), dense_embedding_sum[index]), (
|
||||
"dense-embedding result not match"
|
||||
)
|
||||
|
||||
|
||||
def _check_sparse_embedding(data, check_tokens=False):
|
||||
expected_weights = [
|
||||
{"token_id": 32, "weight": 0.0552978515625, "token": "?"},
|
||||
{"token_id": 70, "weight": 0.09808349609375, "token": " the"},
|
||||
{"token_id": 83, "weight": 0.08154296875, "token": " is"},
|
||||
{"token_id": 111, "weight": 0.11810302734375, "token": " of"},
|
||||
{"token_id": 4865, "weight": 0.1171875, "token": " What"},
|
||||
{"token_id": 9942, "weight": 0.292236328125, "token": " France"},
|
||||
{"token_id": 10323, "weight": 0.2802734375, "token": " capital"},
|
||||
]
|
||||
expected_embed = {x["token_id"]: x for x in expected_weights}
|
||||
|
||||
assert len(data) == len(expected_embed)
|
||||
for entry in data:
|
||||
expected_val = expected_embed[_get_attr_or_val(entry, "token_id")]
|
||||
assert _float_close(
|
||||
expected_val["weight"], _get_attr_or_val(entry, "weight")
|
||||
), f"actual embed {entry} not equal to {expected_val}"
|
||||
if check_tokens:
|
||||
assert expected_val["token"] == _get_attr_or_val(entry, "token"), (
|
||||
f"actual embed {entry} not equal to {expected_val}"
|
||||
)
|
||||
else:
|
||||
assert _get_attr_or_val(entry, "token") is None, (
|
||||
f"{entry} should not return token"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def server():
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--hf_overrides",
|
||||
model_config["hf_overrides"],
|
||||
"--io-processor-plugin",
|
||||
model_config["plugin"],
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(model_config["model_name"], args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"return_tokens",
|
||||
[True, False],
|
||||
)
|
||||
async def test_bge_m3_sparse_plugin_online(
|
||||
server: RemoteOpenAIServer, return_tokens: bool
|
||||
):
|
||||
"""Test BGE-M3 sparse plugin in online mode via API."""
|
||||
request_payload = {
|
||||
"model": model_config["model_name"],
|
||||
"task": "plugin",
|
||||
"data": {"input": model_config["test_input"], "return_tokens": return_tokens},
|
||||
}
|
||||
|
||||
ret = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json=request_payload,
|
||||
)
|
||||
|
||||
response = ret.json()
|
||||
|
||||
# Verify the request response is in the correct format
|
||||
parsed_response: object = IOProcessorResponse(**response).data
|
||||
assert parsed_response
|
||||
|
||||
# Verify the output is formatted as expected for this plugin
|
||||
assert _get_attr_or_val(parsed_response, "data")
|
||||
assert len(_get_attr_or_val(parsed_response, "data")) > 0
|
||||
|
||||
data_entry = _get_attr_or_val(parsed_response, "data")[0]
|
||||
assert _get_attr_or_val(data_entry, "object") == "dense&sparse"
|
||||
assert _get_attr_or_val(data_entry, "sparse_embedding")
|
||||
|
||||
# Verify sparse embedding format
|
||||
sparse_embedding = _get_attr_or_val(data_entry, "sparse_embedding")
|
||||
assert isinstance(sparse_embedding, list)
|
||||
_check_sparse_embedding(sparse_embedding, return_tokens)
|
||||
|
||||
# Verify dense embedding format
|
||||
dense_embedding = _get_attr_or_val(data_entry, "dense_embedding")
|
||||
assert isinstance(dense_embedding, list)
|
||||
_check_dense_embedding(dense_embedding)
|
||||
|
||||
# Verify usage information
|
||||
usage = _get_attr_or_val(parsed_response, "usage")
|
||||
assert usage, f"usage not found for {parsed_response}"
|
||||
assert _get_attr_or_val(usage, "prompt_tokens") > 0
|
||||
assert _get_attr_or_val(usage, "total_tokens") == _get_attr_or_val(
|
||||
usage, "prompt_tokens"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"return_tokens",
|
||||
[True, False],
|
||||
)
|
||||
def test_bge_m3_sparse_plugin_offline(vllm_runner, return_tokens: bool):
|
||||
"""Test BGE-M3 sparse plugin in offline mode."""
|
||||
prompt = {
|
||||
"data": {
|
||||
"input": model_config["test_input"],
|
||||
"return_tokens": return_tokens,
|
||||
}
|
||||
}
|
||||
|
||||
with vllm_runner(
|
||||
model_config["model_name"],
|
||||
runner="pooling",
|
||||
enforce_eager=True,
|
||||
max_num_seqs=32,
|
||||
io_processor_plugin=model_config["plugin"],
|
||||
hf_overrides=json.loads(model_config["hf_overrides"]),
|
||||
default_torch_num_threads=1,
|
||||
) as llm_runner:
|
||||
llm = llm_runner.get_llm()
|
||||
pooler_output = llm.encode(prompt, pooling_task="plugin")
|
||||
|
||||
outputs = pooler_output[0]
|
||||
|
||||
# Verify output structure
|
||||
assert hasattr(outputs, "outputs")
|
||||
response = outputs.outputs
|
||||
assert hasattr(response, "data")
|
||||
assert len(response.data) == 1
|
||||
# Verify response data
|
||||
for i, output in enumerate(response.data):
|
||||
# Each output should have sparse embeddings
|
||||
sparse_embedding = output.sparse_embedding
|
||||
assert isinstance(sparse_embedding, list)
|
||||
_check_sparse_embedding(sparse_embedding, return_tokens)
|
||||
dense_embedding = output.dense_embedding
|
||||
assert isinstance(dense_embedding, list)
|
||||
_check_dense_embedding(dense_embedding)
|
||||
|
||||
# Verify usage
|
||||
assert response.usage.prompt_tokens > 0
|
||||
assert response.usage.total_tokens == response.usage.prompt_tokens
|
||||
|
||||
|
||||
def test_bge_m3_sparse_plugin_offline_multiple_inputs(vllm_runner):
|
||||
"""Test BGE-M3 sparse plugin with multiple inputs in offline mode."""
|
||||
prompts = {
|
||||
"data": {
|
||||
"input": [
|
||||
"What is the capital of France?",
|
||||
"What is the capital of Germany?",
|
||||
"What is the capital of Spain?",
|
||||
],
|
||||
"return_tokens": True,
|
||||
}
|
||||
}
|
||||
|
||||
with vllm_runner(
|
||||
model_config["model_name"],
|
||||
runner="pooling",
|
||||
enforce_eager=True,
|
||||
max_num_seqs=32,
|
||||
io_processor_plugin=model_config["plugin"],
|
||||
hf_overrides=json.loads(model_config["hf_overrides"]),
|
||||
default_torch_num_threads=1,
|
||||
) as llm_runner:
|
||||
llm = llm_runner.get_llm()
|
||||
pooler_output = llm.encode(prompts, pooling_task="plugin")
|
||||
|
||||
outputs = pooler_output[0]
|
||||
|
||||
# Verify output structure
|
||||
assert hasattr(outputs, "outputs")
|
||||
response = outputs.outputs
|
||||
assert hasattr(response, "data")
|
||||
assert len(response.data) == 3
|
||||
for i, output in enumerate(response.data):
|
||||
# Each output should have sparse embeddings
|
||||
sparse_embedding = output.sparse_embedding
|
||||
assert isinstance(sparse_embedding, list)
|
||||
dense_embedding = output.dense_embedding
|
||||
assert isinstance(dense_embedding, list)
|
||||
_check_dense_embedding(dense_embedding, i)
|
||||
|
||||
# Verify usage
|
||||
assert response.usage.prompt_tokens > 0
|
||||
assert response.usage.total_tokens == response.usage.prompt_tokens
|
||||
@@ -1,222 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from typing import TypedDict
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.pooling.protocol import IOProcessorResponse
|
||||
|
||||
|
||||
# Test configuration for ColBERT query plugin
|
||||
class ModelConfig(TypedDict):
|
||||
model_name: str
|
||||
plugin: str
|
||||
query_input: str
|
||||
document_input: str
|
||||
hf_overrides: str
|
||||
embedding_dim: int
|
||||
query_maxlen: int
|
||||
|
||||
|
||||
model_config: ModelConfig = {
|
||||
"model_name": "jinaai/jina-colbert-v2",
|
||||
"plugin": "colbert_query_plugin",
|
||||
"query_input": "What is machine learning?",
|
||||
"document_input": "Machine learning is a subset of artificial intelligence.",
|
||||
"hf_overrides": json.dumps({"architectures": ["ColBERTJinaRobertaModel"]}),
|
||||
"embedding_dim": 128,
|
||||
"query_maxlen": 32,
|
||||
}
|
||||
|
||||
|
||||
def _get_attr_or_val(obj: object | dict, key: str):
|
||||
if isinstance(obj, dict) and key in obj:
|
||||
return obj[key]
|
||||
return getattr(obj, key, None)
|
||||
|
||||
|
||||
def _check_token_embeddings(entry, expected_input_type: str):
|
||||
assert _get_attr_or_val(entry, "object") == "embedding"
|
||||
assert _get_attr_or_val(entry, "input_type") == expected_input_type
|
||||
|
||||
embedding = _get_attr_or_val(entry, "embedding")
|
||||
assert isinstance(embedding, list) and len(embedding) > 0
|
||||
for token_embedding in embedding:
|
||||
assert isinstance(token_embedding, list)
|
||||
assert len(token_embedding) == model_config["embedding_dim"]
|
||||
return embedding
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--trust-remote-code",
|
||||
"--hf_overrides",
|
||||
model_config["hf_overrides"],
|
||||
"--io-processor-plugin",
|
||||
model_config["plugin"],
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(model_config["model_name"], args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
def _post_pooling(server: RemoteOpenAIServer, data: dict):
|
||||
request_payload = {
|
||||
"model": model_config["model_name"],
|
||||
"task": "plugin",
|
||||
"data": data,
|
||||
}
|
||||
ret = requests.post(server.url_for("pooling"), json=request_payload)
|
||||
ret.raise_for_status()
|
||||
response = ret.json()
|
||||
parsed_response: object = IOProcessorResponse(**response).data
|
||||
assert parsed_response
|
||||
return parsed_response
|
||||
|
||||
|
||||
def test_colbert_query_plugin_query_online(server: RemoteOpenAIServer):
|
||||
"""Queries are expanded to exactly query_maxlen token vectors."""
|
||||
parsed_response = _post_pooling(
|
||||
server, {"input": model_config["query_input"], "input_type": "query"}
|
||||
)
|
||||
|
||||
data = _get_attr_or_val(parsed_response, "data")
|
||||
assert len(data) == 1
|
||||
|
||||
embedding = _check_token_embeddings(data[0], "query")
|
||||
assert len(embedding) == model_config["query_maxlen"]
|
||||
|
||||
usage = _get_attr_or_val(parsed_response, "usage")
|
||||
assert _get_attr_or_val(usage, "prompt_tokens") == model_config["query_maxlen"]
|
||||
|
||||
|
||||
def test_colbert_query_plugin_document_online(server: RemoteOpenAIServer):
|
||||
"""Documents return one vector per token, with no mask expansion."""
|
||||
parsed_response = _post_pooling(
|
||||
server, {"input": model_config["document_input"], "input_type": "document"}
|
||||
)
|
||||
|
||||
data = _get_attr_or_val(parsed_response, "data")
|
||||
assert len(data) == 1
|
||||
|
||||
embedding = _check_token_embeddings(data[0], "document")
|
||||
# No query expansion: number of vectors tracks the input length.
|
||||
assert len(embedding) != model_config["query_maxlen"]
|
||||
|
||||
usage = _get_attr_or_val(parsed_response, "usage")
|
||||
assert _get_attr_or_val(usage, "prompt_tokens") == len(embedding)
|
||||
|
||||
|
||||
def test_colbert_query_plugin_missing_input_type_online(server: RemoteOpenAIServer):
|
||||
"""input_type is required; omitting it is rejected."""
|
||||
request_payload = {
|
||||
"model": model_config["model_name"],
|
||||
"task": "plugin",
|
||||
"data": {"input": model_config["document_input"]},
|
||||
}
|
||||
ret = requests.post(server.url_for("pooling"), json=request_payload)
|
||||
assert ret.status_code == 400
|
||||
|
||||
|
||||
def test_colbert_query_plugin_batch_online(server: RemoteOpenAIServer):
|
||||
"""A list input returns one entry per prompt."""
|
||||
queries = ["What is machine learning?", "What is deep learning?"]
|
||||
parsed_response = _post_pooling(server, {"input": queries, "input_type": "query"})
|
||||
|
||||
data = _get_attr_or_val(parsed_response, "data")
|
||||
assert len(data) == len(queries)
|
||||
for i, entry in enumerate(data):
|
||||
assert _get_attr_or_val(entry, "index") == i
|
||||
embedding = _check_token_embeddings(entry, "query")
|
||||
assert len(embedding) == model_config["query_maxlen"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("input_type", ["query", "document"])
|
||||
def test_colbert_query_plugin_offline(vllm_runner, input_type: str):
|
||||
"""Test the ColBERT query plugin in offline mode."""
|
||||
input_text = (
|
||||
model_config["query_input"]
|
||||
if input_type == "query"
|
||||
else model_config["document_input"]
|
||||
)
|
||||
prompt = {
|
||||
"data": {
|
||||
"input": input_text,
|
||||
"input_type": input_type,
|
||||
}
|
||||
}
|
||||
|
||||
with vllm_runner(
|
||||
model_config["model_name"],
|
||||
runner="pooling",
|
||||
enforce_eager=True,
|
||||
max_num_seqs=32,
|
||||
trust_remote_code=True,
|
||||
io_processor_plugin=model_config["plugin"],
|
||||
hf_overrides=json.loads(model_config["hf_overrides"]),
|
||||
default_torch_num_threads=1,
|
||||
) as llm_runner:
|
||||
llm = llm_runner.get_llm()
|
||||
pooler_output = llm.encode(prompt, pooling_task="plugin")
|
||||
|
||||
response = pooler_output[0].outputs
|
||||
assert len(response.data) == 1
|
||||
|
||||
embedding = _check_token_embeddings(response.data[0], input_type)
|
||||
if input_type == "query":
|
||||
assert len(embedding) == model_config["query_maxlen"]
|
||||
else:
|
||||
assert len(embedding) != model_config["query_maxlen"]
|
||||
|
||||
assert response.usage.prompt_tokens == len(embedding)
|
||||
assert response.usage.total_tokens == response.usage.prompt_tokens
|
||||
|
||||
|
||||
def test_colbert_query_plugin_offline_multiple_inputs(vllm_runner):
|
||||
"""Test the ColBERT query plugin with multiple inputs in offline mode."""
|
||||
queries = [
|
||||
"What is machine learning?",
|
||||
"What is deep learning?",
|
||||
"Why?",
|
||||
]
|
||||
prompts = {
|
||||
"data": {
|
||||
"input": queries,
|
||||
"input_type": "query",
|
||||
}
|
||||
}
|
||||
|
||||
with vllm_runner(
|
||||
model_config["model_name"],
|
||||
runner="pooling",
|
||||
enforce_eager=True,
|
||||
max_num_seqs=32,
|
||||
trust_remote_code=True,
|
||||
io_processor_plugin=model_config["plugin"],
|
||||
hf_overrides=json.loads(model_config["hf_overrides"]),
|
||||
default_torch_num_threads=1,
|
||||
) as llm_runner:
|
||||
llm = llm_runner.get_llm()
|
||||
pooler_output = llm.encode(prompts, pooling_task="plugin")
|
||||
|
||||
response = pooler_output[0].outputs
|
||||
assert len(response.data) == len(queries)
|
||||
|
||||
for i, entry in enumerate(response.data):
|
||||
assert entry.index == i
|
||||
embedding = _check_token_embeddings(entry, "query")
|
||||
assert len(embedding) == model_config["query_maxlen"]
|
||||
|
||||
expected_tokens = model_config["query_maxlen"] * len(queries)
|
||||
assert response.usage.prompt_tokens == expected_tokens
|
||||
assert response.usage.total_tokens == response.usage.prompt_tokens
|
||||
@@ -1,154 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import importlib.util
|
||||
import io
|
||||
|
||||
import imagehash
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.pooling.protocol import IOProcessorResponse
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
importlib.util.find_spec("terratorch") is None,
|
||||
reason="terratorch unavailable while PyPI has `lightning` quarantined; see #41376",
|
||||
)
|
||||
|
||||
models_config = {
|
||||
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11": {
|
||||
"image_url": "https://huggingface.co/christian-pinto/Prithvi-EO-2.0-300M-TL-VLLM/resolve/main/valencia_example_2024-10-26.tiff", # noqa: E501
|
||||
"out_hash": "aa6d92ad25926a5e",
|
||||
"plugin": "prithvi_to_tiff",
|
||||
},
|
||||
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-BurnScars": {
|
||||
"image_url": "https://huggingface.co/ibm-nasa-geospatial/Prithvi-EO-2.0-300M-BurnScars/resolve/main/examples/subsetted_512x512_HLS.S30.T10SEH.2018190.v1.4_merged.tif", # noqa: E501
|
||||
"out_hash": "c07f4f602da73552",
|
||||
"plugin": "prithvi_to_tiff",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _compute_image_hash(base64_data: str) -> str:
|
||||
# Decode the base64 output and create image from byte stream
|
||||
decoded_image = base64.b64decode(base64_data)
|
||||
image = Image.open(io.BytesIO(decoded_image))
|
||||
|
||||
# Compute perceptual hash of the output image
|
||||
return str(imagehash.phash(image))
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def server(model_name, plugin):
|
||||
args = [
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--enforce-eager",
|
||||
"--skip-tokenizer-init",
|
||||
# Limit the maximum number of parallel requests
|
||||
# to avoid the model going OOM in CI.
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--io-processor-plugin",
|
||||
plugin,
|
||||
"--enable-mm-embeds",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(model_name, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, image_url, plugin, expected_hash",
|
||||
[
|
||||
(model_name, config["image_url"], config["plugin"], config["out_hash"])
|
||||
for model_name, config in models_config.items()
|
||||
],
|
||||
)
|
||||
async def test_prithvi_mae_plugin_online(
|
||||
server: RemoteOpenAIServer,
|
||||
model_name: str,
|
||||
image_url: str | dict,
|
||||
plugin: str,
|
||||
expected_hash: str,
|
||||
):
|
||||
request_payload_url = {
|
||||
"data": {
|
||||
"data": image_url,
|
||||
"data_format": "url",
|
||||
"image_format": "tiff",
|
||||
"out_data_format": "b64_json",
|
||||
},
|
||||
"priority": 0,
|
||||
"model": model_name,
|
||||
"softmax": False,
|
||||
}
|
||||
|
||||
ret = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json=request_payload_url,
|
||||
)
|
||||
|
||||
response = ret.json()
|
||||
|
||||
# verify the request response is in the correct format
|
||||
parsed_response: IOProcessorResponse = IOProcessorResponse(**response)
|
||||
assert parsed_response
|
||||
|
||||
# verify the output is formatted as expected for this plugin
|
||||
plugin_data = parsed_response.data
|
||||
assert all(plugin_data.get(attr) for attr in ["type", "format", "data"])
|
||||
|
||||
# Compute the output image hash and compare it against the expected hash
|
||||
image_hash = _compute_image_hash(plugin_data["data"])
|
||||
assert image_hash == expected_hash, (
|
||||
f"Image hash mismatch: expected {expected_hash}, got {image_hash}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, image_url, plugin, expected_hash",
|
||||
[
|
||||
(model_name, config["image_url"], config["plugin"], config["out_hash"])
|
||||
for model_name, config in models_config.items()
|
||||
],
|
||||
)
|
||||
def test_prithvi_mae_plugin_offline(
|
||||
vllm_runner, model_name: str, image_url: str | dict, plugin: str, expected_hash: str
|
||||
):
|
||||
img_data = dict(
|
||||
data=image_url,
|
||||
data_format="url",
|
||||
image_format="tiff",
|
||||
out_data_format="b64_json",
|
||||
)
|
||||
|
||||
prompt = dict(data=img_data)
|
||||
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
runner="pooling",
|
||||
skip_tokenizer_init=True,
|
||||
enable_mm_embeds=True,
|
||||
enforce_eager=True,
|
||||
# Limit the maximum number of parallel requests
|
||||
# to avoid the model going OOM in CI.
|
||||
max_num_seqs=32,
|
||||
io_processor_plugin=plugin,
|
||||
default_torch_num_threads=1,
|
||||
) as llm_runner:
|
||||
pooler_output = llm_runner.get_llm().encode(prompt, pooling_task="plugin")
|
||||
|
||||
output = pooler_output[0].outputs
|
||||
|
||||
# verify the output is formatted as expected for this plugin
|
||||
assert all(hasattr(output, attr) for attr in ["type", "format", "data"])
|
||||
|
||||
# Compute the output image hash and compare it against the expected hash
|
||||
image_hash = _compute_image_hash(output.data)
|
||||
assert image_hash == expected_hash, (
|
||||
f"Image hash mismatch: expected {expected_hash}, got {image_hash}"
|
||||
)
|
||||
Reference in New Issue
Block a user