[Mini-VLLM] Repair test suite after SM86 cut
- Restore example connector, test helpers, and dropped patch tests - Guard MLA metadata import in the example connector - Point the PLE test at common.ops.ple after upstream #60021 move - Cut tests for removed features: tools, DRY, epd proxy, STT batch, diffusion scheduler, INC/Quark PLE cases, mamba-2 mixer Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,18 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import GenerateModelInfo
|
||||
|
||||
from .ppl_utils import wikitext_ppl_test
|
||||
|
||||
MODELS = [
|
||||
GenerateModelInfo("google/gemma-2b", hf_ppl=21.48524284362793),
|
||||
GenerateModelInfo("google/gemma-2-2b", hf_ppl=102.59290313720703),
|
||||
GenerateModelInfo("google/gemma-3-4b-it", hf_ppl=27.79648208618164),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
|
||||
wikitext_ppl_test(hf_runner, vllm_runner, model_info)
|
||||
@@ -1,24 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import GenerateModelInfo
|
||||
|
||||
from .ppl_utils import wikitext_ppl_test
|
||||
|
||||
MODELS = [GenerateModelInfo("openai-community/gpt2-large", hf_ppl=19.457056045532227)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
|
||||
bf16_ppl = wikitext_ppl_test(hf_runner, vllm_runner, model_info)
|
||||
fp32_ppl = wikitext_ppl_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
vllm_extra_kwargs={"hf_overrides": {"head_dtype": "float32"}},
|
||||
)
|
||||
|
||||
differ = ((fp32_ppl - bf16_ppl) / bf16_ppl) * 100
|
||||
print("fp32 head difference (%):", differ)
|
||||
assert differ < 0
|
||||
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
|
||||
|
||||
def float_embeddings(response: EmbeddingResponse) -> list[list[float]]:
|
||||
embeddings = []
|
||||
for d in response.data:
|
||||
assert isinstance(d.embedding, list)
|
||||
embeddings.append(d.embedding)
|
||||
return embeddings
|
||||
|
||||
|
||||
def run_embedding_correctness_test(
|
||||
hf_model: "HfRunner",
|
||||
inputs: list[str],
|
||||
vllm_outputs: Sequence[list[float]],
|
||||
dimensions: int | None = None,
|
||||
):
|
||||
hf_outputs = hf_model.encode(inputs)
|
||||
if dimensions:
|
||||
hf_outputs = matryoshka_fy(hf_outputs, dimensions)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
def correctness_test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info: EmbedModelInfo,
|
||||
example_prompts,
|
||||
vllm_extra_kwargs=None,
|
||||
hf_model_callback=None,
|
||||
):
|
||||
pytest.skip("Debug only, ci prefers to use mteb test.")
|
||||
|
||||
# The example_prompts has ending "\n", for example:
|
||||
# "Write a short story about a robot that dreams for the first time.\n"
|
||||
# sentence_transformers will strip the input texts, see:
|
||||
# https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
|
||||
# This makes the input_ids different between hf_model and vllm_model.
|
||||
# So we need to strip the input texts to avoid test failing.
|
||||
example_prompts = [str(s).strip() for s in example_prompts]
|
||||
|
||||
vllm_extra_kwargs = vllm_extra_kwargs or {}
|
||||
vllm_extra_kwargs["dtype"] = model_info.dtype
|
||||
|
||||
if model_info.hf_overrides is not None:
|
||||
vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides
|
||||
|
||||
with vllm_runner(
|
||||
model_info.name, runner="pooling", max_model_len=None, **vllm_extra_kwargs
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.embed(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model_info.name,
|
||||
dtype=model_info.hf_dtype,
|
||||
is_sentence_transformer=True,
|
||||
) as hf_model:
|
||||
if hf_model_callback is not None:
|
||||
hf_model_callback(hf_model)
|
||||
|
||||
run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
|
||||
|
||||
|
||||
async def run_client_embeddings(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
queries: list[str],
|
||||
instruction: str = "",
|
||||
) -> list[list[float]]:
|
||||
outputs = await client.embeddings.create(
|
||||
model=model_name,
|
||||
input=[instruction + q for q in queries],
|
||||
)
|
||||
return [data.embedding for data in outputs.data]
|
||||
@@ -1,37 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pytest configuration for vLLM pooling MTEB tests."""
|
||||
|
||||
import gc
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def release_gpu_memory_between_tests():
|
||||
"""Reclaim cached GPU memory between tests in this shard.
|
||||
|
||||
These tests start many vLLM/HF engines sequentially in one pytest
|
||||
process; without explicit reclamation between tests, VRAM fragmentation
|
||||
accumulates and later tests fail at engine startup ("Engine core
|
||||
initialization failed").
|
||||
"""
|
||||
yield
|
||||
gc.collect()
|
||||
if torch.accelerator.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
if current_platform.is_rocm():
|
||||
return
|
||||
try:
|
||||
wait_for_memory_to_settle()
|
||||
except ValueError as e:
|
||||
# Longer-lived (class/module-scoped) engine fixtures may legitimately
|
||||
# still hold VRAM; nothing further to reclaim in that case.
|
||||
logger.info("Failed to clean GPU memory: %s", e)
|
||||
@@ -1,279 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import mteb
|
||||
import numpy as np
|
||||
import torch
|
||||
from mteb.models import ModelMeta
|
||||
from mteb.types import Array
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
import tests.ci_envs as ci_envs
|
||||
from tests.models.utils import (
|
||||
EmbedModelInfo,
|
||||
check_embeddings_close,
|
||||
get_vllm_extra_kwargs,
|
||||
)
|
||||
|
||||
# Most embedding models on the STS12 task (See #17175):
|
||||
# - Model implementation and minor changes in tensor dtype
|
||||
# results in differences less than 1e-4
|
||||
# - Different model results in differences more than 1e-3
|
||||
# 5e-4 is a good tolerance threshold
|
||||
MTEB_EMBED_TASKS = ["STS12"]
|
||||
MTEB_EMBED_TOL = 5e-4
|
||||
|
||||
|
||||
_empty_model_meta = ModelMeta(
|
||||
loader=None,
|
||||
name="vllm/model",
|
||||
revision="1",
|
||||
release_date=None,
|
||||
languages=None,
|
||||
framework=[],
|
||||
similarity_fn_name=None,
|
||||
n_parameters=None,
|
||||
memory_usage_mb=None,
|
||||
max_tokens=None,
|
||||
embed_dim=None,
|
||||
license=None,
|
||||
open_weights=None,
|
||||
public_training_code=None,
|
||||
public_training_data=None,
|
||||
use_instructions=None,
|
||||
training_datasets=None,
|
||||
modalities=["text"], # 'image' can be added to evaluate multimodal models
|
||||
)
|
||||
|
||||
|
||||
class MtebEmbedMixin(mteb.EncoderProtocol):
|
||||
mteb_model_meta = _empty_model_meta
|
||||
|
||||
def similarity(
|
||||
self,
|
||||
embeddings1: np.ndarray,
|
||||
embeddings2: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
# Cosine similarity
|
||||
norm1 = np.linalg.norm(embeddings1, axis=1, keepdims=True)
|
||||
norm2 = np.linalg.norm(embeddings2, axis=1, keepdims=True)
|
||||
sim = np.dot(embeddings1, embeddings2.T) / (norm1 * norm2.T)
|
||||
return sim
|
||||
|
||||
def similarity_pairwise(
|
||||
self,
|
||||
embeddings1: Array,
|
||||
embeddings2: Array,
|
||||
) -> Array:
|
||||
# Cosine similarity
|
||||
norm1 = np.linalg.norm(embeddings1, axis=1, keepdims=True)
|
||||
norm2 = np.linalg.norm(embeddings2, axis=1, keepdims=True)
|
||||
sim = np.sum(embeddings1 * embeddings2, axis=1) / (
|
||||
norm1.flatten() * norm2.flatten()
|
||||
)
|
||||
return sim
|
||||
|
||||
|
||||
class HfMtebEncoder(MtebEmbedMixin):
|
||||
def __init__(self, model):
|
||||
self.model = model
|
||||
|
||||
def encode(
|
||||
self,
|
||||
inputs: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
sentences = [text for batch in inputs for text in batch["text"]]
|
||||
return self.model.encode(sentences)
|
||||
|
||||
|
||||
class VllmMtebEncoder(MtebEmbedMixin):
|
||||
def __init__(self, vllm_model, prompt_prefix: str | None = None):
|
||||
self.llm = vllm_model
|
||||
self.rng = np.random.default_rng(seed=42)
|
||||
self.prompt_prefix = prompt_prefix
|
||||
|
||||
def encode(
|
||||
self,
|
||||
inputs: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
# Hoping to discover potential scheduling
|
||||
# issues by randomizing the order.
|
||||
sentences = [
|
||||
self.prompt_prefix + text if self.prompt_prefix else text
|
||||
for batch in inputs
|
||||
for text in batch["text"]
|
||||
]
|
||||
r = self.rng.permutation(len(sentences))
|
||||
sentences = [sentences[i] for i in r]
|
||||
outputs = self.llm.embed(sentences, use_tqdm=False)
|
||||
embeds = np.array(outputs)
|
||||
embeds = embeds[np.argsort(r)]
|
||||
return embeds
|
||||
|
||||
|
||||
class OpenAIClientMtebEncoder(MtebEmbedMixin):
|
||||
def __init__(self, model_name: str, client):
|
||||
self.model_name = model_name
|
||||
self.client = client
|
||||
self.rng = np.random.default_rng(seed=42)
|
||||
|
||||
def encode(
|
||||
self,
|
||||
inputs: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
# Hoping to discover potential scheduling
|
||||
# issues by randomizing the order.
|
||||
sentences = [text for batch in inputs for text in batch["text"]]
|
||||
r = self.rng.permutation(len(sentences))
|
||||
sentences = [sentences[i] for i in r]
|
||||
|
||||
embeddings = self.client.embeddings.create(
|
||||
model=self.model_name, input=sentences
|
||||
)
|
||||
outputs = [d.embedding for d in embeddings.data]
|
||||
embeds = np.array(outputs)
|
||||
embeds = embeds[np.argsort(r)]
|
||||
return embeds
|
||||
|
||||
|
||||
def run_mteb_embed_task(encoder: mteb.EncoderProtocol, tasks):
|
||||
tasks = mteb.get_tasks(tasks=tasks)
|
||||
results = mteb.evaluate(
|
||||
encoder,
|
||||
tasks,
|
||||
cache=None,
|
||||
show_progress_bar=False,
|
||||
)
|
||||
|
||||
main_score = results[0].scores["test"][0]["main_score"]
|
||||
return main_score
|
||||
|
||||
|
||||
def mteb_test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info: EmbedModelInfo,
|
||||
vllm_extra_kwargs=None,
|
||||
hf_model_callback=None,
|
||||
atol: float | None = None,
|
||||
prompt_prefix: str | None = None,
|
||||
vllm_model_callback=None,
|
||||
):
|
||||
if atol is None:
|
||||
atol = (
|
||||
model_info.mteb_tol if model_info.mteb_tol is not None else MTEB_EMBED_TOL
|
||||
)
|
||||
vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs)
|
||||
|
||||
# Test embed_dims, isnan and whether to use normalize
|
||||
example_prompts = ["The chef prepared a delicious meal." * 1000]
|
||||
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=model_info.max_model_len,
|
||||
**vllm_extra_kwargs,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
|
||||
if vllm_model_callback is not None:
|
||||
vllm_model_callback(vllm_model)
|
||||
|
||||
# Confirm whether vllm is using the correct architecture
|
||||
if model_info.architecture:
|
||||
assert model_info.architecture in model_config.architectures
|
||||
|
||||
# Confirm whether the important configs in model_config are correct.
|
||||
pooler_config = model_config.pooler_config
|
||||
if model_info.seq_pooling_type is not None:
|
||||
assert pooler_config.seq_pooling_type == model_info.seq_pooling_type
|
||||
if model_info.tok_pooling_type is not None:
|
||||
assert pooler_config.tok_pooling_type == model_info.tok_pooling_type
|
||||
if model_info.attn_type is not None:
|
||||
assert model_config.attn_type == model_info.attn_type
|
||||
if model_info.is_prefix_caching_supported is not None:
|
||||
assert (
|
||||
model_config.is_prefix_caching_supported
|
||||
== model_info.is_prefix_caching_supported
|
||||
)
|
||||
if model_info.is_chunked_prefill_supported is not None:
|
||||
assert (
|
||||
model_config.is_chunked_prefill_supported
|
||||
== model_info.is_chunked_prefill_supported
|
||||
)
|
||||
|
||||
vllm_main_score = run_mteb_embed_task(
|
||||
VllmMtebEncoder(vllm_model, prompt_prefix=prompt_prefix), MTEB_EMBED_TASKS
|
||||
)
|
||||
vllm_dtype = vllm_model.llm.llm_engine.model_config.dtype
|
||||
head_dtype = model_config.head_dtype
|
||||
|
||||
# Test embedding_size, isnan and whether to use normalize.
|
||||
# Apply the same prompt_prefix used for scoring so this check compares
|
||||
# identical effective inputs: HF's SentenceTransformer.encode() applies
|
||||
# the model's default prompt (e.g. "Document: "), so vLLM must receive it
|
||||
# too. This mirrors VllmMtebEncoder and changes no production behavior.
|
||||
consistency_prompts = (
|
||||
[prompt_prefix + p for p in example_prompts]
|
||||
if prompt_prefix
|
||||
else example_prompts
|
||||
)
|
||||
vllm_outputs = vllm_model.embed(
|
||||
consistency_prompts,
|
||||
tokenization_kwargs=dict(truncate_prompt_tokens=-1),
|
||||
)
|
||||
outputs_tensor = torch.tensor(vllm_outputs)
|
||||
assert not torch.any(torch.isnan(outputs_tensor))
|
||||
embedding_size = model_config.embedding_size
|
||||
assert torch.tensor(vllm_outputs).shape[-1] == embedding_size
|
||||
|
||||
# Accelerate mteb test by setting
|
||||
# SentenceTransformers mteb score to a constant
|
||||
if model_info.mteb_score is None:
|
||||
with hf_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
is_sentence_transformer=True,
|
||||
dtype=ci_envs.VLLM_CI_HF_DTYPE or model_info.hf_dtype,
|
||||
) as hf_model:
|
||||
# e.g. setting default parameters for the encode method of hf_runner
|
||||
if hf_model_callback is not None:
|
||||
hf_model_callback(hf_model)
|
||||
|
||||
st_main_score = run_mteb_embed_task(
|
||||
HfMtebEncoder(hf_model), MTEB_EMBED_TASKS
|
||||
)
|
||||
st_dtype = next(hf_model.model.parameters()).dtype
|
||||
|
||||
# Check embeddings close to hf outputs
|
||||
hf_outputs = hf_model.encode(example_prompts)
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
else:
|
||||
st_main_score = model_info.mteb_score
|
||||
st_dtype = "Constant"
|
||||
|
||||
print("Model:", model_info.name)
|
||||
print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_main_score)
|
||||
print("SentenceTransformers:", st_dtype, st_main_score)
|
||||
print("Difference:", st_main_score - vllm_main_score)
|
||||
|
||||
# We are not concerned that the vllm mteb results are better
|
||||
# than SentenceTransformers, so we only perform one-sided testing.
|
||||
diff = st_main_score - vllm_main_score
|
||||
assert diff < atol, (
|
||||
f"diff={diff:.6g} tol={atol} model={model_info.name} "
|
||||
f"(st={st_main_score}, vllm={vllm_main_score})"
|
||||
)
|
||||
@@ -1,319 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import mteb
|
||||
import numpy as np
|
||||
import requests
|
||||
import torch
|
||||
from mteb.models import ModelMeta
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import (
|
||||
RerankModelInfo,
|
||||
get_vllm_extra_kwargs,
|
||||
)
|
||||
|
||||
# See #19344
|
||||
MTEB_RERANK_TASKS = ["NFCorpus"]
|
||||
MTEB_RERANK_LANGS = ["eng"]
|
||||
MTEB_RERANK_TOL = 2e-3
|
||||
|
||||
template_home = (
|
||||
Path(__file__).parent.parent.parent.parent.parent
|
||||
/ "examples/pooling/score/template"
|
||||
)
|
||||
|
||||
_empty_model_meta = ModelMeta(
|
||||
loader=None,
|
||||
name="vllm/model",
|
||||
revision="1",
|
||||
release_date=None,
|
||||
languages=None,
|
||||
framework=[],
|
||||
similarity_fn_name=None,
|
||||
n_parameters=None,
|
||||
memory_usage_mb=None,
|
||||
max_tokens=None,
|
||||
embed_dim=None,
|
||||
license=None,
|
||||
open_weights=None,
|
||||
public_training_code=None,
|
||||
public_training_data=None,
|
||||
use_instructions=None,
|
||||
training_datasets=None,
|
||||
modalities=["text"], # 'image' can be added to evaluate multimodal models
|
||||
)
|
||||
|
||||
|
||||
class MtebCrossEncoderMixin(mteb.CrossEncoderProtocol):
|
||||
mteb_model_meta = _empty_model_meta
|
||||
|
||||
|
||||
class VllmMtebCrossEncoder(MtebCrossEncoderMixin):
|
||||
def __init__(self, vllm_model):
|
||||
self.llm = vllm_model
|
||||
self.rng = np.random.default_rng(seed=42)
|
||||
self.chat_template: str | None = getattr(vllm_model, "chat_template", None)
|
||||
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
# Hoping to discover potential scheduling
|
||||
# issues by randomizing the order.
|
||||
r = self.rng.permutation(len(queries))
|
||||
queries = [queries[i] for i in r]
|
||||
corpus = [corpus[i] for i in r]
|
||||
|
||||
outputs = self.llm.score(
|
||||
queries,
|
||||
corpus,
|
||||
use_tqdm=False,
|
||||
chat_template=self.chat_template,
|
||||
tokenization_kwargs={"truncate_prompt_tokens": -1},
|
||||
)
|
||||
scores = np.array(outputs)
|
||||
scores = scores[np.argsort(r)]
|
||||
return scores
|
||||
|
||||
|
||||
class ScoreClientMtebEncoder(MtebCrossEncoderMixin):
|
||||
mteb_model_meta = _empty_model_meta
|
||||
|
||||
def __init__(self, model_name: str, url):
|
||||
self.model_name = model_name
|
||||
self.url = url
|
||||
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
full_corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
outputs = []
|
||||
for query, corpus in zip(queries, full_corpus):
|
||||
outputs.append(self.get_score(query, corpus))
|
||||
|
||||
scores = np.array(outputs)
|
||||
return scores
|
||||
|
||||
def get_score(self, query, corpus):
|
||||
response = requests.post(
|
||||
self.url,
|
||||
json={
|
||||
"model": self.model_name,
|
||||
"queries": query,
|
||||
"documents": corpus,
|
||||
"truncate_prompt_tokens": -1,
|
||||
},
|
||||
).json()
|
||||
return response["data"][0]["score"]
|
||||
|
||||
|
||||
class RerankClientMtebEncoder(ScoreClientMtebEncoder):
|
||||
def get_score(self, query, corpus):
|
||||
response = requests.post(
|
||||
self.url,
|
||||
json={
|
||||
"model": self.model_name,
|
||||
"query": query,
|
||||
"documents": [corpus],
|
||||
"truncate_prompt_tokens": -1,
|
||||
},
|
||||
).json()
|
||||
return response["results"][0]["relevance_score"]
|
||||
|
||||
|
||||
class HFMtebCrossEncoder(MtebCrossEncoderMixin, HfRunner):
|
||||
chat_template: str | None = None
|
||||
|
||||
def __init__(self, model_name: str, dtype: str = "auto", **kwargs: Any) -> None:
|
||||
HfRunner.__init__(
|
||||
self, model_name=model_name, is_cross_encoder=True, dtype=dtype, **kwargs
|
||||
)
|
||||
|
||||
@torch.no_grad
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
if self.chat_template is not None:
|
||||
tokenizer = self.model.tokenizer
|
||||
prompts = []
|
||||
for query, document in zip(queries, corpus):
|
||||
conversation = [
|
||||
{"role": "query", "content": query},
|
||||
{"role": "document", "content": document},
|
||||
]
|
||||
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
tools=None,
|
||||
chat_template=self.chat_template,
|
||||
tokenize=False,
|
||||
)
|
||||
prompts.append(prompt)
|
||||
outputs_list = HfRunner.classify(self, prompts)
|
||||
scores = np.array(outputs_list).squeeze(-1)
|
||||
return scores
|
||||
else:
|
||||
prompts = list(zip(queries, corpus))
|
||||
outputs_tensor = HfRunner.predict(self, prompts, show_progress_bar=False)
|
||||
return outputs_tensor.cpu().numpy()
|
||||
|
||||
|
||||
def run_mteb_rerank(cross_encoder: mteb.CrossEncoderProtocol, tasks, languages):
|
||||
with tempfile.TemporaryDirectory() as prediction_folder:
|
||||
bm25s = mteb.get_model("bm25s")
|
||||
eval_splits = ["test"]
|
||||
|
||||
mteb_tasks: list[mteb.abstasks.AbsTaskRetrieval] = mteb.get_tasks(
|
||||
tasks=tasks, languages=languages, eval_splits=eval_splits
|
||||
)
|
||||
for task in mteb_tasks:
|
||||
if not task.data_loaded:
|
||||
task.load_data()
|
||||
|
||||
mteb.evaluate(
|
||||
bm25s,
|
||||
mteb_tasks,
|
||||
prediction_folder=prediction_folder,
|
||||
show_progress_bar=False,
|
||||
# don't save results for test runs
|
||||
cache=None,
|
||||
overwrite_strategy="always",
|
||||
)
|
||||
|
||||
second_stage_tasks = []
|
||||
for task in mteb_tasks:
|
||||
second_stage_tasks.append(
|
||||
task.convert_to_reranking(
|
||||
prediction_folder,
|
||||
top_k=10,
|
||||
)
|
||||
)
|
||||
|
||||
results = mteb.evaluate(
|
||||
cross_encoder,
|
||||
second_stage_tasks,
|
||||
show_progress_bar=False,
|
||||
cache=None,
|
||||
)
|
||||
main_score = results[0].scores["test"][0]["main_score"]
|
||||
return main_score
|
||||
|
||||
|
||||
def mteb_test_rerank_models(
|
||||
vllm_runner,
|
||||
model_info: RerankModelInfo,
|
||||
hf_runner=HFMtebCrossEncoder,
|
||||
vllm_extra_kwargs=None,
|
||||
vllm_mteb_encoder=VllmMtebCrossEncoder,
|
||||
atol: float | None = None,
|
||||
):
|
||||
if atol is None:
|
||||
atol = (
|
||||
model_info.mteb_tol if model_info.mteb_tol is not None else MTEB_RERANK_TOL
|
||||
)
|
||||
vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs)
|
||||
|
||||
# Maybe load chat_template.
|
||||
chat_template: str | None = None
|
||||
if model_info.chat_template_name is not None:
|
||||
chat_template = (template_home / model_info.chat_template_name).read_text()
|
||||
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=None,
|
||||
max_num_seqs=8,
|
||||
**vllm_extra_kwargs,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
vllm_model.chat_template = chat_template
|
||||
|
||||
# Confirm whether vllm is using the correct architecture
|
||||
if model_info.architecture:
|
||||
assert model_info.architecture in model_config.architectures
|
||||
|
||||
# Score API is only enabled for num_labels == 1
|
||||
assert model_config.hf_config.num_labels == 1
|
||||
|
||||
# Confirm whether the important configs in model_config are correct.
|
||||
pooler_config = model_config.pooler_config
|
||||
if model_info.seq_pooling_type is not None:
|
||||
assert pooler_config.seq_pooling_type == model_info.seq_pooling_type
|
||||
if model_info.tok_pooling_type is not None:
|
||||
assert pooler_config.tok_pooling_type == model_info.tok_pooling_type
|
||||
if model_info.attn_type is not None:
|
||||
assert model_config.attn_type == model_info.attn_type
|
||||
if model_info.is_prefix_caching_supported is not None:
|
||||
assert (
|
||||
model_config.is_prefix_caching_supported
|
||||
== model_info.is_prefix_caching_supported
|
||||
)
|
||||
if model_info.is_chunked_prefill_supported is not None:
|
||||
assert (
|
||||
model_config.is_chunked_prefill_supported
|
||||
== model_info.is_chunked_prefill_supported
|
||||
)
|
||||
|
||||
vllm_main_score = run_mteb_rerank(
|
||||
vllm_mteb_encoder(vllm_model),
|
||||
tasks=MTEB_RERANK_TASKS,
|
||||
languages=MTEB_RERANK_LANGS,
|
||||
)
|
||||
vllm_dtype = model_config.dtype
|
||||
head_dtype = model_config.head_dtype
|
||||
|
||||
# Accelerate mteb test by setting
|
||||
# SentenceTransformers mteb score to a constant
|
||||
if model_info.mteb_score is None:
|
||||
with hf_runner(
|
||||
model_info.name, revision=model_info.revision, dtype=model_info.hf_dtype
|
||||
) as hf_model:
|
||||
hf_model.chat_template = chat_template
|
||||
st_main_score = run_mteb_rerank(
|
||||
hf_model,
|
||||
tasks=MTEB_RERANK_TASKS,
|
||||
languages=MTEB_RERANK_LANGS,
|
||||
)
|
||||
st_dtype = next(hf_model.model.model.parameters()).dtype
|
||||
else:
|
||||
st_main_score = model_info.mteb_score
|
||||
st_dtype = "Constant"
|
||||
|
||||
print("Model:", model_info.name)
|
||||
print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_main_score)
|
||||
print("SentenceTransformers:", st_dtype, st_main_score)
|
||||
print("Difference:", st_main_score - vllm_main_score)
|
||||
|
||||
# We are not concerned that the vllm mteb results are better
|
||||
# than SentenceTransformers, so we only perform one-sided testing.
|
||||
diff = st_main_score - vllm_main_score
|
||||
assert diff < atol, (
|
||||
f"diff={diff:.6g} tol={atol} model={model_info.name} "
|
||||
f"(st={st_main_score}, vllm={vllm_main_score})"
|
||||
)
|
||||
@@ -1,123 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
|
||||
from tests.models.utils import (
|
||||
EmbedModelInfo,
|
||||
RerankModelInfo,
|
||||
)
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
from .mteb_score_utils import mteb_test_rerank_models
|
||||
|
||||
MODELS = [
|
||||
########## BertModel
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-base-en",
|
||||
architecture="BertModel",
|
||||
mteb_score=0.779336792,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo("BAAI/bge-base-zh", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo("BAAI/bge-small-en", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo("BAAI/bge-small-zh", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo("BAAI/bge-large-en", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo("BAAI/bge-large-zh", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-large-zh-noinstruct", architecture="BertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-base-en-v1.5", architecture="BertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-base-zh-v1.5", architecture="BertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-small-en-v1.5", architecture="BertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-small-zh-v1.5", architecture="BertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-large-en-v1.5", architecture="BertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-large-zh-v1.5", architecture="BertModel", enable_test=False
|
||||
),
|
||||
########## XLMRobertaModel
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-m3",
|
||||
architecture="XLMRobertaModel",
|
||||
mteb_score=0.787343078,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
########## Qwen2Model
|
||||
EmbedModelInfo(
|
||||
"BAAI/bge-code-v1",
|
||||
architecture="Qwen2Model",
|
||||
mteb_score=0.75724465,
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
# Skip: model's custom tokenizer on HF hub is incompatible with
|
||||
# transformers v5 (sets attrs before super().__init__, triggering
|
||||
# AttributeError on 'verbose' in __getattr__).
|
||||
enable_test=False,
|
||||
),
|
||||
]
|
||||
|
||||
RERANK_MODELS = [
|
||||
########## XLMRobertaForSequenceClassification
|
||||
RerankModelInfo(
|
||||
"BAAI/bge-reranker-base",
|
||||
architecture="XLMRobertaForSequenceClassification",
|
||||
mteb_score=0.32398,
|
||||
mteb_tol=1e-2,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
RerankModelInfo(
|
||||
"BAAI/bge-reranker-large",
|
||||
architecture="XLMRobertaForSequenceClassification",
|
||||
enable_test=False,
|
||||
),
|
||||
RerankModelInfo(
|
||||
"BAAI/bge-reranker-v2-m3",
|
||||
architecture="XLMRobertaForSequenceClassification",
|
||||
enable_test=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_correctness(
|
||||
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
|
||||
) -> None:
|
||||
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(vllm_runner, model_info)
|
||||
@@ -1,138 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import mteb
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import RerankModelInfo
|
||||
|
||||
from .mteb_score_utils import (
|
||||
MtebCrossEncoderMixin,
|
||||
mteb_test_rerank_models,
|
||||
)
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
"BAAI/bge-reranker-v2-gemma",
|
||||
architecture="GemmaForSequenceClassification",
|
||||
hf_overrides={
|
||||
"architectures": ["GemmaForSequenceClassification"],
|
||||
"classifier_from_token": ["Yes"],
|
||||
"method": "no_post_processing",
|
||||
},
|
||||
mteb_score=0.33757,
|
||||
mteb_tol=1e-2,
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
chat_template_name="bge-reranker-v2-gemma.jinja",
|
||||
),
|
||||
]
|
||||
|
||||
PROMPT = "Given a query A and a passage B, determine whether the passage contains an answer to the query by providing a prediction of either 'Yes' or 'No'." # noqa: E501
|
||||
|
||||
|
||||
class GemmaRerankerHfRunner(MtebCrossEncoderMixin, HfRunner):
|
||||
def __init__(
|
||||
self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
|
||||
) -> None:
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
HfRunner.__init__(
|
||||
self,
|
||||
model_name=model_name,
|
||||
auto_cls=AutoModelForCausalLM,
|
||||
dtype=dtype,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
|
||||
self.yes_loc = self.tokenizer.convert_tokens_to_ids("Yes")
|
||||
|
||||
@torch.no_grad
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
def get_inputs(pairs, tokenizer, prompt=None):
|
||||
if prompt is None:
|
||||
prompt = PROMPT
|
||||
|
||||
sep = "\n"
|
||||
prompt_inputs = tokenizer(
|
||||
prompt, return_tensors=None, add_special_tokens=False
|
||||
)["input_ids"]
|
||||
sep_inputs = tokenizer(sep, return_tensors=None, add_special_tokens=False)[
|
||||
"input_ids"
|
||||
]
|
||||
inputs = []
|
||||
for query, passage in pairs:
|
||||
query_inputs = tokenizer(
|
||||
f"A: {query}",
|
||||
return_tensors=None,
|
||||
add_special_tokens=False,
|
||||
truncation=True,
|
||||
)
|
||||
passage_inputs = tokenizer(
|
||||
f"B: {passage}",
|
||||
return_tensors=None,
|
||||
add_special_tokens=False,
|
||||
truncation=True,
|
||||
)
|
||||
item = tokenizer.prepare_for_model(
|
||||
[tokenizer.bos_token_id] + query_inputs["input_ids"],
|
||||
sep_inputs + passage_inputs["input_ids"],
|
||||
truncation="only_second",
|
||||
padding=False,
|
||||
return_attention_mask=False,
|
||||
return_token_type_ids=False,
|
||||
add_special_tokens=False,
|
||||
)
|
||||
item["input_ids"] = item["input_ids"] + sep_inputs + prompt_inputs
|
||||
item["attention_mask"] = [1] * len(item["input_ids"])
|
||||
inputs.append(item)
|
||||
return tokenizer.pad(
|
||||
inputs,
|
||||
padding=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
|
||||
scores = []
|
||||
for query, document in zip(queries, corpus):
|
||||
pairs = [(query, document)]
|
||||
inputs = get_inputs(pairs, self.tokenizer)
|
||||
inputs = inputs.to(self.model.device)
|
||||
_n_tokens = inputs["input_ids"].shape[1]
|
||||
logits = self.model(**inputs, return_dict=True).logits
|
||||
_scores = (
|
||||
logits[:, -1, self.yes_loc]
|
||||
.view(
|
||||
-1,
|
||||
)
|
||||
.float()
|
||||
.sigmoid()
|
||||
)
|
||||
scores.append(_scores[0].item())
|
||||
return torch.Tensor(scores)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(
|
||||
vllm_runner,
|
||||
model_info,
|
||||
hf_runner=GemmaRerankerHfRunner,
|
||||
)
|
||||
@@ -1,39 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import (
|
||||
RerankModelInfo,
|
||||
)
|
||||
|
||||
from .mteb_score_utils import mteb_test_rerank_models
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
"cross-encoder/ms-marco-TinyBERT-L-2-v2",
|
||||
architecture="BertForSequenceClassification",
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
mteb_score=0.32898,
|
||||
mteb_tol=1e-2,
|
||||
),
|
||||
RerankModelInfo(
|
||||
"tomaarsen/Qwen3-Reranker-0.6B-seq-cls",
|
||||
architecture="Qwen3ForSequenceClassification",
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
chat_template_name="qwen3_reranker.jinja",
|
||||
mteb_score=0.33459,
|
||||
mteb_tol=1e-2,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(vllm_runner, model_info)
|
||||
@@ -1,63 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
|
||||
from tests.models.utils import EmbedModelInfo
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
|
||||
MODELS = [
|
||||
########## BertModel
|
||||
EmbedModelInfo(
|
||||
"intfloat/e5-small",
|
||||
architecture="BertModel",
|
||||
mteb_score=0.742285423,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo("intfloat/e5-base", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo("intfloat/e5-large", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo(
|
||||
"intfloat/multilingual-e5-small", architecture="BertModel", enable_test=False
|
||||
),
|
||||
########## XLMRobertaModel
|
||||
EmbedModelInfo(
|
||||
"intfloat/multilingual-e5-base",
|
||||
architecture="XLMRobertaModel",
|
||||
mteb_score=0.779325955,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"intfloat/multilingual-e5-large",
|
||||
architecture="XLMRobertaModel",
|
||||
enable_test=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"intfloat/multilingual-e5-large-instruct",
|
||||
architecture="XLMRobertaModel",
|
||||
enable_test=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_correctness(
|
||||
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
|
||||
) -> None:
|
||||
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
|
||||
@@ -1,111 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import mteb
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import RerankModelInfo
|
||||
|
||||
from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models
|
||||
|
||||
mxbai_rerank_hf_overrides = {
|
||||
"architectures": ["Qwen2ForSequenceClassification"],
|
||||
"classifier_from_token": ["0", "1"],
|
||||
"method": "from_2_way_softmax",
|
||||
}
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
"mixedbread-ai/mxbai-rerank-base-v2",
|
||||
architecture="Qwen2ForSequenceClassification",
|
||||
hf_overrides=mxbai_rerank_hf_overrides,
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
chat_template_name="mxbai_rerank_v2.jinja",
|
||||
mteb_score=0.33651,
|
||||
mteb_tol=1e-2,
|
||||
enable_test=True,
|
||||
),
|
||||
RerankModelInfo(
|
||||
"mixedbread-ai/mxbai-rerank-large-v2",
|
||||
architecture="Qwen2ForSequenceClassification",
|
||||
hf_overrides=mxbai_rerank_hf_overrides,
|
||||
chat_template_name="mxbai_rerank_v2.jinja",
|
||||
enable_test=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class MxbaiRerankerHfRunner(MtebCrossEncoderMixin, HfRunner):
|
||||
def __init__(
|
||||
self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
|
||||
) -> None:
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
HfRunner.__init__(
|
||||
self,
|
||||
model_name=model_name,
|
||||
auto_cls=AutoModelForCausalLM,
|
||||
dtype=dtype,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
|
||||
self.yes_loc = self.tokenizer.convert_tokens_to_ids("1")
|
||||
self.no_loc = self.tokenizer.convert_tokens_to_ids("0")
|
||||
|
||||
@torch.no_grad
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
tokenizer = self.tokenizer
|
||||
prompts = []
|
||||
for query, document in zip(queries, corpus):
|
||||
conversation = [
|
||||
{"role": "query", "content": query},
|
||||
{"role": "document", "content": document},
|
||||
]
|
||||
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
tools=None,
|
||||
chat_template=self.chat_template,
|
||||
tokenize=False,
|
||||
)
|
||||
prompts.append(prompt)
|
||||
|
||||
def compute_logits(inputs):
|
||||
logits = self.model(**inputs).logits[:, -1, :]
|
||||
yes_logits = logits[:, self.yes_loc]
|
||||
no_logits = logits[:, self.no_loc]
|
||||
logits = yes_logits - no_logits
|
||||
scores = logits.float().sigmoid()
|
||||
return scores
|
||||
|
||||
scores = []
|
||||
for prompt in prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = self.wrap_device(inputs)
|
||||
score = compute_logits(inputs)
|
||||
scores.append(score[0].item())
|
||||
return torch.Tensor(scores)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(vllm_runner, model_info, hf_runner=MxbaiRerankerHfRunner)
|
||||
@@ -1,54 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling_mteb_test.mteb_embed_utils import (
|
||||
mteb_test_embed_models,
|
||||
)
|
||||
from tests.models.language.pooling_mteb_test.mteb_score_utils import (
|
||||
mteb_test_rerank_models,
|
||||
)
|
||||
from tests.models.utils import (
|
||||
EmbedModelInfo,
|
||||
RerankModelInfo,
|
||||
)
|
||||
|
||||
EMBEDDING_MODELS = [
|
||||
EmbedModelInfo(
|
||||
"nvidia/llama-nemotron-embed-1b-v2",
|
||||
architecture="LlamaBidirectionalModel",
|
||||
mteb_score=0.689164662128673,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
)
|
||||
]
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
"nvidia/llama-nemotron-rerank-1b-v2",
|
||||
architecture="LlamaBidirectionalForSequenceClassification",
|
||||
chat_template_name="nemotron-rerank.jinja",
|
||||
mteb_score=0.33994,
|
||||
mteb_tol=1e-2,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(vllm_runner, model_info)
|
||||
@@ -1,59 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
|
||||
from tests.models.utils import EmbedModelInfo
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
|
||||
MODELS = [
|
||||
EmbedModelInfo(
|
||||
"nomic-ai/nomic-embed-text-v1",
|
||||
architecture="NomicBertModel",
|
||||
# Fixme:
|
||||
# Update nomic-embed code to support the latest
|
||||
# HF version and remove revision set.
|
||||
revision="720244025c1a7e15661a174c63cce63c8218e52b",
|
||||
mteb_score=0.737568559,
|
||||
mteb_tol=2e-3,
|
||||
enable_test=True,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"nomic-ai/nomic-embed-text-v1.5",
|
||||
architecture="NomicBertModel",
|
||||
enable_test=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"nomic-ai/CodeRankEmbed", architecture="NomicBertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"nomic-ai/nomic-embed-text-v2-moe",
|
||||
architecture="NomicBertModel",
|
||||
mteb_score=0.715488912,
|
||||
mteb_tol=2e-3,
|
||||
enable_test=True,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_correctness(
|
||||
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
|
||||
) -> None:
|
||||
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
|
||||
@@ -1,133 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
from typing import Any
|
||||
|
||||
import mteb
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import RerankModelInfo
|
||||
from tests.utils import multi_gpu_test
|
||||
|
||||
from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models
|
||||
|
||||
qwen3_reranker_hf_overrides = {
|
||||
"architectures": ["Qwen3ForSequenceClassification"],
|
||||
"classifier_from_token": ["no", "yes"],
|
||||
"is_original_qwen3_reranker": True,
|
||||
}
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
"Qwen/Qwen3-Reranker-0.6B",
|
||||
architecture="Qwen3ForSequenceClassification",
|
||||
hf_overrides=qwen3_reranker_hf_overrides,
|
||||
chat_template_name="qwen3_reranker.jinja",
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
mteb_score=0.33459,
|
||||
mteb_tol=1e-2,
|
||||
enable_test=True,
|
||||
),
|
||||
RerankModelInfo(
|
||||
"Qwen/Qwen3-Reranker-4B",
|
||||
architecture="Qwen3ForSequenceClassification",
|
||||
chat_template_name="qwen3_reranker.jinja",
|
||||
hf_overrides=qwen3_reranker_hf_overrides,
|
||||
enable_test=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class Qwen3RerankerHfRunner(MtebCrossEncoderMixin, HfRunner):
|
||||
def __init__(
|
||||
self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
|
||||
) -> None:
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
HfRunner.__init__(
|
||||
self,
|
||||
model_name=model_name,
|
||||
auto_cls=AutoModelForCausalLM,
|
||||
dtype=dtype,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
|
||||
self.token_false_id = self.tokenizer.convert_tokens_to_ids("no")
|
||||
self.token_true_id = self.tokenizer.convert_tokens_to_ids("yes")
|
||||
self.max_length = 40960
|
||||
|
||||
@torch.no_grad
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
tokenizer = self.tokenizer
|
||||
prompts = []
|
||||
for query, document in zip(queries, corpus):
|
||||
conversation = [
|
||||
{"role": "query", "content": query},
|
||||
{"role": "document", "content": document},
|
||||
]
|
||||
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
tools=None,
|
||||
chat_template=self.chat_template,
|
||||
tokenize=False,
|
||||
)
|
||||
prompts.append(prompt)
|
||||
|
||||
def compute_logits(inputs):
|
||||
batch_scores = self.model(**inputs).logits[:, -1, :]
|
||||
true_vector = batch_scores[:, self.token_true_id]
|
||||
false_vector = batch_scores[:, self.token_false_id]
|
||||
batch_scores = torch.stack([false_vector, true_vector], dim=1)
|
||||
batch_scores = torch.nn.functional.log_softmax(batch_scores, dim=1)
|
||||
scores = batch_scores[:, 1].exp()
|
||||
return scores
|
||||
|
||||
scores = []
|
||||
for prompt in prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = self.wrap_device(inputs)
|
||||
score = compute_logits(inputs)
|
||||
scores.append(score[0].item())
|
||||
return torch.Tensor(scores)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(vllm_runner, model_info, hf_runner=Qwen3RerankerHfRunner)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_rerank_models_mteb_tp(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
assert model_info.architecture == "Qwen3ForSequenceClassification"
|
||||
|
||||
vllm_extra_kwargs: dict[str, Any] = {
|
||||
"tensor_parallel_size": 2,
|
||||
}
|
||||
|
||||
mteb_test_rerank_models(
|
||||
vllm_runner,
|
||||
model_info,
|
||||
vllm_extra_kwargs=vllm_extra_kwargs,
|
||||
hf_runner=Qwen3RerankerHfRunner,
|
||||
)
|
||||
@@ -1,103 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
|
||||
from tests.models.utils import EmbedModelInfo
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
|
||||
MODELS = [
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-xs",
|
||||
is_matryoshka=False,
|
||||
architecture="BertModel",
|
||||
mteb_score=0.714927797,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-s",
|
||||
is_matryoshka=False,
|
||||
architecture="BertModel",
|
||||
enable_test=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-m",
|
||||
is_matryoshka=False,
|
||||
architecture="BertModel",
|
||||
enable_test=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-m-long",
|
||||
is_matryoshka=False,
|
||||
architecture="NomicBertModel",
|
||||
mteb_score=0.681146831,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-l",
|
||||
is_matryoshka=False,
|
||||
architecture="BertModel",
|
||||
enable_test=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-m-v1.5",
|
||||
is_matryoshka=True,
|
||||
architecture="BertModel",
|
||||
mteb_score=0.649088363,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-l-v2.0",
|
||||
is_matryoshka=True,
|
||||
architecture="XLMRobertaModel",
|
||||
mteb_score=0.712258299,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Snowflake/snowflake-arctic-embed-m-v2.0",
|
||||
is_matryoshka=True,
|
||||
architecture="GteModel",
|
||||
mteb_score=0.706622444,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_correctness(
|
||||
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
|
||||
) -> None:
|
||||
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
|
||||
@@ -1,41 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import (
|
||||
EmbedModelInfo,
|
||||
)
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
|
||||
# ST models with projector (Dense) layers
|
||||
ST_PROJECTOR_MODELS = [
|
||||
EmbedModelInfo(
|
||||
"TencentBAC/Conan-embedding-v1",
|
||||
architecture="BertModel",
|
||||
mteb_score=0.688611955,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"google/embeddinggemma-300m",
|
||||
architecture="Gemma3TextModel",
|
||||
mteb_score=0.7473819294684156,
|
||||
mteb_tol=2e-3,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", ST_PROJECTOR_MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
|
||||
@@ -1,58 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
|
||||
from tests.models.utils import EmbedModelInfo
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
|
||||
MODELS = [
|
||||
EmbedModelInfo(
|
||||
"voyageai/voyage-4-nano",
|
||||
architecture="VoyageQwen3BidirectionalEmbedModel",
|
||||
enable_test=True,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
hf_overrides={
|
||||
"architectures": ["VoyageQwen3BidirectionalEmbedModel"],
|
||||
"num_labels": 2048,
|
||||
},
|
||||
mteb_score=0.7054,
|
||||
mteb_tol=2e-3,
|
||||
# === MTEB Results ===
|
||||
# STS12: 0.6613
|
||||
# STS13: 0.6906
|
||||
# STS14: 0.6556
|
||||
# STS15: 0.7843
|
||||
# STS16: 0.7340
|
||||
# STSBenchmark: 0.7063
|
||||
# Average score: 0.7054
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
# Encoder-only attention models need enforce_eager=True to avoid
|
||||
# CUDA graph capture issues with piecewise compilation
|
||||
mteb_test_embed_models(
|
||||
hf_runner, vllm_runner, model_info, vllm_extra_kwargs={"enforce_eager": True}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_correctness(
|
||||
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
|
||||
) -> None:
|
||||
correctness_test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
example_prompts,
|
||||
vllm_extra_kwargs={"enforce_eager": True},
|
||||
)
|
||||
@@ -1,95 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import mteb
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import RerankModelInfo
|
||||
|
||||
from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
"polaria-tech/zerank-2-reranker-vllm",
|
||||
architecture="Qwen3ForSequenceClassification",
|
||||
chat_template_name="zerank2.jinja",
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
mteb_score=0.34305,
|
||||
mteb_tol=1e-2,
|
||||
enable_test=True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class ZerankHfRunner(MtebCrossEncoderMixin, HfRunner):
|
||||
# The converted checkpoint has no lm_head, so the reference is the original
|
||||
# causal LM: the logit of the "Yes" token at the last position. The converted
|
||||
# checkpoint returns sigmoid(logit / 5), which ranks documents identically.
|
||||
original_model = "zeroentropy/zerank-2-reranker"
|
||||
|
||||
def __init__(
|
||||
self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
|
||||
) -> None:
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
HfRunner.__init__(
|
||||
self,
|
||||
model_name=self.original_model,
|
||||
auto_cls=AutoModelForCausalLM,
|
||||
dtype=dtype,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(
|
||||
self.original_model, padding_side="left"
|
||||
)
|
||||
self.token_true_id = self.tokenizer.convert_tokens_to_ids("Yes")
|
||||
|
||||
@torch.no_grad
|
||||
def predict(
|
||||
self,
|
||||
inputs1: DataLoader[mteb.types.BatchedInput],
|
||||
inputs2: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
queries = [text for batch in inputs1 for text in batch["text"]]
|
||||
corpus = [text for batch in inputs2 for text in batch["text"]]
|
||||
|
||||
tokenizer = self.tokenizer
|
||||
prompts = []
|
||||
for query, document in zip(queries, corpus):
|
||||
conversation = [
|
||||
{"role": "query", "content": query},
|
||||
{"role": "document", "content": document},
|
||||
]
|
||||
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
conversation=conversation,
|
||||
tools=None,
|
||||
chat_template=self.chat_template,
|
||||
tokenize=False,
|
||||
)
|
||||
prompts.append(prompt)
|
||||
|
||||
scores = []
|
||||
for prompt in prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = self.wrap_device(inputs)
|
||||
logits = self.model(**inputs).logits[:, -1, :]
|
||||
scores.append(logits[0, self.token_true_id].item())
|
||||
return torch.Tensor(scores)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(vllm_runner, model_info, hf_runner=ZerankHfRunner)
|
||||
@@ -1,96 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
MODELS = {
|
||||
"Gemma3nForConditionalGeneration": "google/gemma-3n-E2B-it",
|
||||
"Gemma4ForConditionalGeneration": "google/gemma-4-E2B-it",
|
||||
}
|
||||
|
||||
# Seconds. Spread wide enough that no two clips can produce the same number of
|
||||
# mel frames: equal-length clips stack normally and never reach the ragged path.
|
||||
CLIP_DURATIONS = (1.0, 3.5, 7.0, 12.0)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def ragged_clips():
|
||||
"""One asset sliced to several durations, so only length varies."""
|
||||
audio, sample_rate = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
|
||||
clips = []
|
||||
for duration in CLIP_DURATIONS:
|
||||
num_samples = int(duration * sample_rate)
|
||||
if num_samples > len(audio):
|
||||
pytest.skip(f"asset is shorter than {duration}s")
|
||||
clips.append((audio[:num_samples], sample_rate))
|
||||
|
||||
assert len({len(clip) for clip, _ in clips}) == len(clips)
|
||||
return clips
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("arch", sorted(MODELS))
|
||||
@pytest.mark.parametrize("max_tokens", [8])
|
||||
def test_variable_length_audio_batching(
|
||||
vllm_runner,
|
||||
ragged_clips,
|
||||
arch: str,
|
||||
max_tokens: int,
|
||||
) -> None:
|
||||
"""Test batching of requests with different audio durations.
|
||||
|
||||
Audio features are unpadded per item so a multimodal cache entry does not
|
||||
depend on the batch it was first processed in, and
|
||||
`MultiModalFieldConfig.batched` stacks items only when their shapes agree.
|
||||
A batch of differently sized clips therefore reaches the model as a list,
|
||||
which used to take EngineCore down with
|
||||
|
||||
AttributeError: 'list' object has no attribute 'squeeze'
|
||||
|
||||
failing every in-flight request with EngineDeadError.
|
||||
"""
|
||||
from transformers import AutoProcessor
|
||||
|
||||
model_name = MODELS[arch]
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info(arch)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
# The two architectures spell the audio placeholder differently; ask the
|
||||
# processor rather than hardcoding either.
|
||||
processor = AutoProcessor.from_pretrained(model_name)
|
||||
prompt = processor.apply_chat_template(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "audio", "audio": None},
|
||||
{"type": "text", "text": "Transcribe this audio."},
|
||||
],
|
||||
}
|
||||
],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
dtype="bfloat16",
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
) as vllm_model:
|
||||
# One call so the clips are scheduled into the same forward pass;
|
||||
# issued separately they never reach the ragged path.
|
||||
outputs = vllm_model.generate_greedy(
|
||||
[prompt] * len(ragged_clips),
|
||||
max_tokens,
|
||||
audios=[[clip] for clip in ragged_clips],
|
||||
)
|
||||
|
||||
assert len(outputs) == len(ragged_clips)
|
||||
for output in outputs:
|
||||
assert len(output[1]) > 0, "Expected non-empty output"
|
||||
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
|
||||
models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]
|
||||
|
||||
|
||||
def base_prompt(modalities_str: str) -> str:
|
||||
return f"<|im_start|>user {modalities_str}\nDescribe what you see from these items.<|im_end|><|im_start|>assistant\n" # noqa: E501
|
||||
|
||||
|
||||
INTERLEAVED_PROMPT = base_prompt("<image><video><image>\n")
|
||||
NONINTERLEAVED_PROMPT = base_prompt("<image><image><video>\n")
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("model", models)
|
||||
@pytest.mark.parametrize("dtype", ["float16"])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
def test_models(vllm_runner, model, dtype: str, max_tokens: int) -> None:
|
||||
"""This is a simple test to check if interleaved and non-interleaved prompts
|
||||
give the same result.
|
||||
"""
|
||||
image_cherry = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
|
||||
image_stop = convert_image_mode(ImageAsset("stop_sign").pil_image, "RGB")
|
||||
images = [image_cherry, image_stop]
|
||||
video = VideoAsset(name="baby_reading", num_frames=16).np_ndarrays
|
||||
|
||||
inputs = [
|
||||
(
|
||||
[INTERLEAVED_PROMPT],
|
||||
[images],
|
||||
[video],
|
||||
),
|
||||
(
|
||||
[NONINTERLEAVED_PROMPT],
|
||||
[images],
|
||||
[video],
|
||||
),
|
||||
]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="generate",
|
||||
dtype=dtype,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
max_model_len=32768,
|
||||
max_num_seqs=2,
|
||||
tensor_parallel_size=1,
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs_per_case = [
|
||||
vllm_model.generate_greedy(
|
||||
prompts, max_tokens, images=images, videos=videos
|
||||
)
|
||||
for prompts, images, videos in inputs
|
||||
]
|
||||
|
||||
all_results = [output[0][1] for output in vllm_outputs_per_case]
|
||||
outputs = [
|
||||
(total_str, total_str.find("assistant\n") + len("assistant\n"))
|
||||
for total_str in all_results
|
||||
]
|
||||
prompt_lengths = [prompt_len for _, prompt_len in outputs]
|
||||
generated_strs = [total_str[prompt_len:] for total_str, prompt_len in outputs]
|
||||
interleaved_prompt_len, noninterleaved_prompt_len = prompt_lengths
|
||||
interleaved_output_str, noninterleaved_output_str = generated_strs
|
||||
|
||||
# The two prompts are identical except for the order of modality tokens.
|
||||
assert interleaved_prompt_len == noninterleaved_prompt_len
|
||||
|
||||
# The two generated strings should be different because of the
|
||||
# interleaved modality tokens.
|
||||
assert interleaved_output_str != noninterleaved_output_str
|
||||
@@ -1,296 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
from collections.abc import Sequence
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.image import convert_image_mode, rescale_image_size
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
from ....conftest import (
|
||||
IMAGE_ASSETS,
|
||||
HfRunner,
|
||||
PromptAudioInput,
|
||||
PromptImageInput,
|
||||
VllmRunner,
|
||||
)
|
||||
from ....utils import large_gpu_test
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<|user|>\n<|image_1|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
|
||||
"cherry_blossom": "<|user|>\n<|image_1|>\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n", # noqa: E501
|
||||
}
|
||||
)
|
||||
HF_MULTIIMAGE_IMAGE_PROMPT = (
|
||||
"<|user|>\n<|image_1|>\n<|image_2|>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501
|
||||
)
|
||||
|
||||
model_path = hf_api().snapshot_download("microsoft/Phi-4-multimodal-instruct")
|
||||
# Since the vision-lora and speech-lora co-exist with the base model,
|
||||
# we have to manually specify the path of the lora weights.
|
||||
vision_lora_path = os.path.join(model_path, "vision-lora")
|
||||
speech_question = os.path.join(
|
||||
model_path, "examples", "what_is_shown_in_this_image.wav"
|
||||
)
|
||||
models = [model_path]
|
||||
|
||||
|
||||
def vllm_to_hf_output(
|
||||
vllm_output: tuple[list[int], str, SampleLogprobs | None], model: str
|
||||
):
|
||||
"""Sanitize vllm output to be comparable with hf output."""
|
||||
_, output_str, out_logprobs = vllm_output
|
||||
|
||||
output_str_without_image = re.sub(r"(<\|image_\d+\|>)+", "", output_str)
|
||||
assert output_str_without_image[0] == " "
|
||||
output_str_without_image = output_str_without_image[1:]
|
||||
|
||||
hf_output_str = output_str_without_image + "<|end|><|endoftext|>"
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model)
|
||||
hf_output_ids = tokenizer.encode(output_str_without_image)
|
||||
assert hf_output_ids[0] == 1
|
||||
hf_output_ids = hf_output_ids[1:]
|
||||
|
||||
return hf_output_ids, hf_output_str, out_logprobs
|
||||
|
||||
|
||||
target_dtype = "half"
|
||||
IMAGE_SIZE_FACTOR_GROUPS = (
|
||||
(1.0,),
|
||||
(1.0, 1.0, 1.0),
|
||||
(0.25, 0.5, 1.0),
|
||||
)
|
||||
|
||||
|
||||
def run_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
inputs: Sequence[tuple[list[str], PromptImageInput, PromptAudioInput | None]],
|
||||
model: str,
|
||||
*,
|
||||
max_model_len: int,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
mm_limit: int,
|
||||
tensor_parallel_size: int,
|
||||
distributed_executor_backend: str | None = None,
|
||||
):
|
||||
"""Inference result should be the same between hf and vllm.
|
||||
|
||||
All the image fixtures for the test are from IMAGE_ASSETS.
|
||||
For huggingface runner, we provide the PIL images as input.
|
||||
For vllm runner, we provide MultiModalDataDict objects
|
||||
and corresponding MultiModalConfig as input.
|
||||
Note, the text input is also adjusted to abide by vllm contract.
|
||||
The text output is sanitized to be able to compare with hf.
|
||||
"""
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
# max_model_len should be greater than image_feature_size
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="generate",
|
||||
max_model_len=max_model_len,
|
||||
max_num_seqs=2,
|
||||
dtype=dtype,
|
||||
limit_mm_per_prompt={"image": mm_limit},
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enable_lora=True,
|
||||
max_lora_rank=320,
|
||||
gpu_memory_utilization=0.8, # set to 0.8 to avoid OOM in CI
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
lora_request = LoRARequest("vision", 1, vision_lora_path)
|
||||
vllm_outputs_per_case = [
|
||||
vllm_model.generate_greedy_logprobs(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
images=images,
|
||||
audios=audios,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
for prompts, images, audios in inputs
|
||||
]
|
||||
|
||||
# This error occurs inside `get_peft_model`
|
||||
# FIXME: https://huggingface.co/microsoft/Phi-4-multimodal-instruct/discussions/75
|
||||
pytest.skip("HF impl is not compatible with current transformers")
|
||||
|
||||
hf_model_kwargs = {"_attn_implementation": "sdpa"}
|
||||
with hf_runner(model, dtype=dtype, model_kwargs=hf_model_kwargs) as hf_model:
|
||||
hf_processor = hf_model.processor
|
||||
eos_token_id = hf_processor.tokenizer.eos_token_id
|
||||
|
||||
def patch_hf_processor(
|
||||
*args, text="", images=None, audio=None, sampling_rate=None, **kwargs
|
||||
):
|
||||
audios = None
|
||||
if audio is not None and sampling_rate is not None:
|
||||
audios = [(audio, sampling_rate)]
|
||||
return hf_processor(
|
||||
*args, text=text, images=images, audios=audios, **kwargs
|
||||
)
|
||||
|
||||
hf_model.processor = patch_hf_processor
|
||||
|
||||
hf_outputs_per_case = [
|
||||
hf_model.generate_greedy_logprobs_limit(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
images=images,
|
||||
audios=audios,
|
||||
eos_token_id=eos_token_id,
|
||||
num_logits_to_keep=0,
|
||||
)
|
||||
for prompts, images, audios in inputs
|
||||
]
|
||||
|
||||
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", models)
|
||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||
@pytest.mark.parametrize("max_model_len", [12800])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [10])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model,
|
||||
dtype: str,
|
||||
max_model_len: int,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
|
||||
inputs_per_image = [
|
||||
(
|
||||
[prompt for _ in size_factors],
|
||||
[rescale_image_size(image, factor) for factor in size_factors],
|
||||
None,
|
||||
)
|
||||
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
|
||||
for image, prompt in zip(images, HF_IMAGE_PROMPTS)
|
||||
]
|
||||
|
||||
run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
inputs_per_image,
|
||||
model,
|
||||
dtype=dtype,
|
||||
max_model_len=max_model_len,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
mm_limit=1,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
|
||||
|
||||
@large_gpu_test(min_gb=48)
|
||||
@pytest.mark.parametrize("model", models)
|
||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||
@pytest.mark.parametrize("max_model_len", [25600])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [10])
|
||||
def test_multi_images_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model,
|
||||
dtype: str,
|
||||
max_model_len: int,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
|
||||
inputs_per_case = [
|
||||
(
|
||||
[HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
|
||||
[
|
||||
[rescale_image_size(image, factor) for image in images]
|
||||
for factor in size_factors
|
||||
],
|
||||
None,
|
||||
)
|
||||
for size_factors in IMAGE_SIZE_FACTOR_GROUPS
|
||||
]
|
||||
|
||||
run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
inputs_per_case,
|
||||
model,
|
||||
dtype=dtype,
|
||||
max_model_len=max_model_len,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
mm_limit=2,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", models)
|
||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||
@pytest.mark.parametrize("max_model_len", [12800])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [10])
|
||||
def test_vision_speech_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model,
|
||||
dtype: str,
|
||||
max_model_len: int,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
# use the example speech question so that the model outputs are reasonable
|
||||
audio = load_audio(speech_question, sr=None)
|
||||
image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
|
||||
|
||||
inputs_vision_speech = [
|
||||
(
|
||||
["<|user|><|image_1|><|audio_1|><|end|><|assistant|>"],
|
||||
[image],
|
||||
[audio],
|
||||
),
|
||||
]
|
||||
|
||||
run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
inputs_vision_speech,
|
||||
model,
|
||||
dtype=dtype,
|
||||
max_model_len=max_model_len,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
mm_limit=1,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
@@ -1,229 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from ....conftest import AUDIO_ASSETS, AudioTestAssets, VllmRunner
|
||||
from ....utils import RemoteOpenAIServer
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
|
||||
MODEL_NAME = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
|
||||
|
||||
AUDIO_PROMPTS = AUDIO_ASSETS.prompts(
|
||||
{
|
||||
"mary_had_lamb": "Transcribe this into English.",
|
||||
"winning_call": "What is happening in this audio clip?",
|
||||
}
|
||||
)
|
||||
|
||||
MULTI_AUDIO_PROMPT = "Describe each of the audios above."
|
||||
|
||||
AudioTuple = tuple[np.ndarray, int]
|
||||
|
||||
VLLM_PLACEHOLDER = "<|audio|>"
|
||||
HF_PLACEHOLDER = "<|audio|>"
|
||||
|
||||
CHUNKED_PREFILL_KWARGS = {
|
||||
"enable_chunked_prefill": True,
|
||||
"max_num_seqs": 2,
|
||||
# Use a very small limit to exercise chunked prefill.
|
||||
"max_num_batched_tokens": 16,
|
||||
}
|
||||
|
||||
|
||||
def params_kwargs_to_cli_args(params_kwargs: dict[str, Any]) -> list[str]:
|
||||
"""Convert kwargs to CLI args."""
|
||||
args = []
|
||||
for key, value in params_kwargs.items():
|
||||
if isinstance(value, bool):
|
||||
if value:
|
||||
args.append(f"--{key.replace('_', '-')}")
|
||||
else:
|
||||
args.append(f"--{key.replace('_', '-')}={value}")
|
||||
return args
|
||||
|
||||
|
||||
@pytest.fixture(
|
||||
params=[
|
||||
pytest.param({}, marks=pytest.mark.cpu_model),
|
||||
pytest.param(CHUNKED_PREFILL_KWARGS),
|
||||
]
|
||||
)
|
||||
def server(request, audio_assets: AudioTestAssets):
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--enforce-eager",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"audio": len(audio_assets)}),
|
||||
"--trust-remote-code",
|
||||
] + params_kwargs_to_cli_args(request.param)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME, args, env_dict={"VLLM_AUDIO_FETCH_TIMEOUT": "30"}
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
def _get_prompt(audio_count, question, placeholder):
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
|
||||
placeholder = f"{placeholder}\n" * audio_count
|
||||
|
||||
return tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": f"{placeholder}{question}"}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
|
||||
|
||||
def run_multi_audio_test(
|
||||
vllm_runner: type[VllmRunner],
|
||||
prompts_and_audios: list[tuple[str, list[AudioTuple]]],
|
||||
model: str,
|
||||
*,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
**kwargs,
|
||||
):
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={
|
||||
"audio": max((len(audio) for _, audio in prompts_and_audios))
|
||||
},
|
||||
**kwargs,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
[prompt for prompt, _ in prompts_and_audios],
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
audios=[audios for _, audios in prompts_and_audios],
|
||||
)
|
||||
|
||||
# The HuggingFace model doesn't support multiple audios yet, so
|
||||
# just assert that some tokens were generated.
|
||||
assert all(tokens for tokens, *_ in vllm_outputs)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.parametrize(
|
||||
"vllm_kwargs",
|
||||
[
|
||||
pytest.param({}, marks=pytest.mark.cpu_model),
|
||||
pytest.param(CHUNKED_PREFILL_KWARGS),
|
||||
],
|
||||
)
|
||||
def test_models_with_multiple_audios(
|
||||
vllm_runner,
|
||||
audio_assets: AudioTestAssets,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
vllm_kwargs: dict,
|
||||
) -> None:
|
||||
vllm_prompt = _get_prompt(len(audio_assets), MULTI_AUDIO_PROMPT, VLLM_PLACEHOLDER)
|
||||
run_multi_audio_test(
|
||||
vllm_runner,
|
||||
[(vllm_prompt, [audio.audio_and_sample_rate for audio in audio_assets])],
|
||||
MODEL_NAME,
|
||||
dtype=dtype,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
**vllm_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
def test_variable_length_audio_batching(
|
||||
vllm_runner,
|
||||
audio_assets: AudioTestAssets,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
) -> None:
|
||||
"""Test batching of requests with different audio durations.
|
||||
|
||||
This exercises the variable-length tensor handling in
|
||||
MultiModalFlatField._reduce_data() which was buggy before
|
||||
https://github.com/vllm-project/vllm/issues/31658 was fixed.
|
||||
"""
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(MODEL_NAME)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
# Create prompts with single audio each (different durations)
|
||||
prompts_and_audios = []
|
||||
for audio, question in zip(audio_assets, AUDIO_PROMPTS):
|
||||
prompt = _get_prompt(1, question, VLLM_PLACEHOLDER)
|
||||
prompts_and_audios.append((prompt, [audio.audio_and_sample_rate]))
|
||||
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
dtype=dtype,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
) as vllm_model:
|
||||
# Generate for all prompts in a single batch
|
||||
# This triggers the variable-length batching code path
|
||||
outputs = vllm_model.generate_greedy(
|
||||
[prompt for prompt, _ in prompts_and_audios],
|
||||
max_tokens,
|
||||
audios=[audios for _, audios in prompts_and_audios],
|
||||
)
|
||||
|
||||
# Verify outputs were generated for each request
|
||||
assert len(outputs) == len(prompts_and_audios)
|
||||
for output in outputs:
|
||||
assert len(output[1]) > 0, "Expected non-empty output"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_serving(client, audio_assets: AudioTestAssets):
|
||||
"""Exercises online serving with/without chunked prefill enabled."""
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*[
|
||||
{"type": "audio_url", "audio_url": {"url": audio.url}}
|
||||
for audio in audio_assets
|
||||
],
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"What's happening in these {len(audio_assets)} audio clips?", # noqa: E501
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
chat_completion = await client.chat.completions.create(
|
||||
model=MODEL_NAME, messages=messages, max_tokens=10
|
||||
)
|
||||
|
||||
assert len(chat_completion.choices) == 1
|
||||
choice = chat_completion.choices[0]
|
||||
assert choice.finish_reason == "length"
|
||||
@@ -1,204 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from mistral_common.protocol.instruct.chunk import AudioChunk, TextChunk
|
||||
from mistral_common.protocol.instruct.messages import UserMessage
|
||||
from mistral_common.tokens.tokenizers.audio import Audio
|
||||
from transformers import VoxtralForConditionalGeneration
|
||||
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
from ....conftest import AudioTestAssets
|
||||
from ....utils import RemoteOpenAIServer
|
||||
from ...utils import check_logprobs_close
|
||||
from .test_ultravox import MULTI_AUDIO_PROMPT, run_multi_audio_test
|
||||
from .vlm_utils import model_utils
|
||||
|
||||
MODEL_NAME = "mistralai/Voxtral-Mini-3B-2507"
|
||||
MISTRAL_FORMAT_ARGS = [
|
||||
"--tokenizer_mode",
|
||||
"mistral",
|
||||
"--config_format",
|
||||
"mistral",
|
||||
"--load_format",
|
||||
"mistral",
|
||||
]
|
||||
|
||||
|
||||
def _get_prompt(audio_assets: AudioTestAssets, question: str) -> list[int]:
|
||||
"""Build a token-ID prompt via mistral_common for vLLM offline inference."""
|
||||
tokenizer = MistralTokenizer.from_pretrained(MODEL_NAME)
|
||||
|
||||
audios = [
|
||||
Audio.from_file(str(asset.get_local_path()), strict=False)
|
||||
for asset in audio_assets
|
||||
]
|
||||
audio_chunks = [AudioChunk.from_audio(audio) for audio in audios]
|
||||
|
||||
messages = [
|
||||
UserMessage(content=[*audio_chunks, TextChunk(text=question)]).to_openai()
|
||||
]
|
||||
return tokenizer.apply_chat_template(messages=messages)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_models_with_multiple_audios(
|
||||
vllm_runner,
|
||||
audio_assets: AudioTestAssets,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
vllm_prompt = _get_prompt(audio_assets, MULTI_AUDIO_PROMPT)
|
||||
run_multi_audio_test(
|
||||
vllm_runner,
|
||||
[(vllm_prompt, [a.audio_and_sample_rate for a in audio_assets])], # type: ignore[list-item]
|
||||
MODEL_NAME,
|
||||
dtype=dtype,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
tokenizer_mode="mistral",
|
||||
)
|
||||
|
||||
|
||||
def test_online_serving(vllm_runner, audio_assets: AudioTestAssets):
|
||||
"""Two-layer accuracy and serving validation using Mistral format.
|
||||
|
||||
1. Offline vLLM greedy output (runs first to avoid CUDA fork issues
|
||||
with multiprocessing - see vlm_utils/core.py).
|
||||
2. Online OpenAI-compatible API output must match offline — validates
|
||||
that the serving path (chat template, audio encoding, tokenization)
|
||||
does not corrupt anything.
|
||||
|
||||
Steps run sequentially so each releases the GPU before the next starts.
|
||||
"""
|
||||
question = f"What's happening in these {len(audio_assets)} audio clips?"
|
||||
max_tokens = 10
|
||||
audio_data = [asset.audio_and_sample_rate for asset in audio_assets]
|
||||
|
||||
vllm_prompt = _get_prompt(audio_assets, question)
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
tokenizer_mode="mistral",
|
||||
config_format="mistral",
|
||||
load_format="mistral",
|
||||
limit_mm_per_prompt={"audio": len(audio_assets)},
|
||||
) as vllm_model:
|
||||
offline_outputs = vllm_model.generate_greedy(
|
||||
[vllm_prompt],
|
||||
max_tokens,
|
||||
audios=[audio_data],
|
||||
)
|
||||
|
||||
offline_text = offline_outputs[0][1]
|
||||
assert offline_text, "Offline vLLM inference produced empty output"
|
||||
|
||||
def _asset_to_openai_chunk(asset):
|
||||
audio = Audio.from_file(str(asset.get_local_path()), strict=False)
|
||||
audio.format = "wav"
|
||||
return AudioChunk.from_audio(audio).to_openai()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*[_asset_to_openai_chunk(a) for a in audio_assets],
|
||||
{"type": "text", "text": question},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
server_args = [
|
||||
"--enforce-eager",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"audio": len(audio_assets)}),
|
||||
*MISTRAL_FORMAT_ARGS,
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
server_args,
|
||||
env_dict={"VLLM_AUDIO_FETCH_TIMEOUT": "30"},
|
||||
) as remote_server:
|
||||
client = remote_server.get_client()
|
||||
completion = client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=0,
|
||||
)
|
||||
|
||||
assert len(completion.choices) == 1
|
||||
choice = completion.choices[0]
|
||||
assert choice.finish_reason == "length"
|
||||
assert choice.message.content == offline_text, (
|
||||
f"Online serving output does not match offline inference.\n"
|
||||
f" Online: {choice.message.content!r}\n"
|
||||
f" Offline: {offline_text!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_hf_reference(hf_runner, vllm_runner, audio_assets: AudioTestAssets):
|
||||
"""Compare vLLM Mistral-format output against HF Transformers reference.
|
||||
|
||||
Instead of requiring an exact text match (which is brittle across
|
||||
attention backends), we compare per-token logprobs using the standard
|
||||
check_logprobs_close helper: when tokens diverge at a position, each
|
||||
runner's chosen token must appear in the other's top-k logprobs.
|
||||
"""
|
||||
question = f"What's happening in these {len(audio_assets)} audio clips?"
|
||||
max_tokens = 10
|
||||
num_logprobs = 5
|
||||
audio_data = [asset.audio_and_sample_rate for asset in audio_assets]
|
||||
|
||||
vllm_prompt = _get_prompt(audio_assets, question)
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
tokenizer_mode="mistral",
|
||||
config_format="mistral",
|
||||
load_format="mistral",
|
||||
limit_mm_per_prompt={"audio": len(audio_assets)},
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
[vllm_prompt],
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
audios=[audio_data],
|
||||
)
|
||||
assert vllm_outputs[0][1], "vLLM inference produced empty output"
|
||||
|
||||
with hf_runner(
|
||||
MODEL_NAME,
|
||||
dtype="half",
|
||||
auto_cls=VoxtralForConditionalGeneration,
|
||||
) as hf_model:
|
||||
hf_model = model_utils.voxtral_patch_hf_runner(hf_model)
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
[question],
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
audios=[audio_data],
|
||||
)
|
||||
assert hf_outputs[0][1], "HF Transformers produced empty output"
|
||||
|
||||
print(
|
||||
f"HF Reference Comparison\n"
|
||||
f" vLLM: {vllm_outputs[0][1]!r}\n"
|
||||
f" HF: {hf_outputs[0][1]!r}"
|
||||
)
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=vllm_outputs,
|
||||
outputs_1_lst=hf_outputs,
|
||||
name_0="vllm",
|
||||
name_1="hf",
|
||||
)
|
||||
@@ -0,0 +1,207 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Core test implementation to be shared across modalities."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from transformers.models.auto.auto_factory import _BaseAutoModelClass
|
||||
|
||||
from vllm.config.model import RunnerOption
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .....conftest import HfRunner, VllmRunner
|
||||
from ....registry import HF_EXAMPLE_MODELS
|
||||
from .types import PromptWithMultiModalInput, RunnerOutput
|
||||
|
||||
|
||||
def run_test(
|
||||
*,
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
inputs: list[PromptWithMultiModalInput],
|
||||
model: str,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
enforce_eager: bool,
|
||||
max_model_len: int,
|
||||
max_num_seqs: int,
|
||||
hf_output_post_proc: Callable[[RunnerOutput, str], Any] | None,
|
||||
vllm_output_post_proc: Callable[[RunnerOutput, str], Any] | None,
|
||||
auto_cls: type[_BaseAutoModelClass],
|
||||
use_tokenizer_eos: bool,
|
||||
comparator: Callable[..., None],
|
||||
get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None,
|
||||
stop_str: list[str] | None,
|
||||
limit_mm_per_prompt: dict[str, int],
|
||||
vllm_runner_kwargs: dict[str, Any] | None,
|
||||
hf_model_kwargs: dict[str, Any] | None,
|
||||
hf_processor: Callable[[str], Any] | None,
|
||||
patch_hf_runner: Callable[[HfRunner], HfRunner] | None,
|
||||
runner: RunnerOption = "auto",
|
||||
distributed_executor_backend: str | None = None,
|
||||
tensor_parallel_size: int = 1,
|
||||
vllm_embeddings: torch.Tensor | None = None,
|
||||
):
|
||||
"""Modality agnostic test executor for comparing HF/vLLM outputs."""
|
||||
# In the case of embeddings, vLLM takes separate input tensors
|
||||
vllm_inputs = vllm_embeddings if vllm_embeddings is not None else inputs
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
# Disable other modalities to save memory
|
||||
default_limits = {"image": 0, "video": 0, "audio": 0}
|
||||
limit_mm_per_prompt = default_limits | limit_mm_per_prompt
|
||||
|
||||
vllm_outputs_per_mm = []
|
||||
hf_outputs_per_mm = []
|
||||
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
|
||||
vllm_runner_kwargs_: dict[str, Any] = {"mm_processor_cache_gb": 0}
|
||||
if model_info.tokenizer:
|
||||
vllm_runner_kwargs_["tokenizer_name"] = model_info.tokenizer
|
||||
if model_info.tokenizer_mode:
|
||||
vllm_runner_kwargs_["tokenizer_mode"] = model_info.tokenizer_mode
|
||||
if model_info.hf_overrides:
|
||||
vllm_runner_kwargs_["hf_overrides"] = model_info.hf_overrides
|
||||
if model_info.require_embed_inputs:
|
||||
for k in ("skip_tokenizer_init", "enable_prompt_embeds", "enable_mm_embeds"):
|
||||
vllm_runner_kwargs_[k] = model_info.require_embed_inputs
|
||||
if not model_info.enable_prefix_caching:
|
||||
vllm_runner_kwargs_["enable_prefix_caching"] = False
|
||||
|
||||
if vllm_runner_kwargs:
|
||||
vllm_runner_kwargs_.update(vllm_runner_kwargs)
|
||||
|
||||
# Avoid passing limit_mm_per_prompt twice when vllm_runner_kwargs
|
||||
# already contains it (e.g. gemma4 sets it via vllm_runner_kwargs).
|
||||
if "limit_mm_per_prompt" in vllm_runner_kwargs_:
|
||||
limit_mm_per_prompt = vllm_runner_kwargs_.pop("limit_mm_per_prompt")
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=max_model_len,
|
||||
max_num_seqs=max_num_seqs,
|
||||
dtype=dtype,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enforce_eager=enforce_eager,
|
||||
runner=runner,
|
||||
**vllm_runner_kwargs_,
|
||||
) as vllm_model:
|
||||
tokenizer = vllm_model.llm.get_tokenizer()
|
||||
|
||||
vllm_kwargs: dict[str, Any] = {}
|
||||
if get_stop_token_ids is not None:
|
||||
vllm_kwargs["stop_token_ids"] = get_stop_token_ids(tokenizer)
|
||||
if stop_str:
|
||||
vllm_kwargs["stop"] = stop_str
|
||||
|
||||
for prompts, image_data, video_data, audio_data in vllm_inputs:
|
||||
mm_data = dict(images=image_data, videos=video_data, audios=audio_data)
|
||||
vllm_kwargs_with_mm_data = vllm_kwargs | mm_data
|
||||
vllm_output = vllm_model.generate_greedy_logprobs(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
**vllm_kwargs_with_mm_data,
|
||||
)
|
||||
vllm_outputs_per_mm.append(vllm_output)
|
||||
|
||||
hf_runner_kwargs: dict[str, Any] = {}
|
||||
if model_info.tokenizer:
|
||||
hf_runner_kwargs["tokenizer_name"] = model_info.tokenizer
|
||||
if hf_processor is not None:
|
||||
hf_runner_kwargs["processor"] = hf_processor(model)
|
||||
|
||||
hf_model = hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=auto_cls,
|
||||
model_kwargs=hf_model_kwargs,
|
||||
**hf_runner_kwargs,
|
||||
)
|
||||
|
||||
# Some models need to patch things like the model processor, e.g., internvl
|
||||
if patch_hf_runner is not None:
|
||||
hf_model = patch_hf_runner(hf_model)
|
||||
|
||||
with hf_model, torch.no_grad():
|
||||
tokenizer = hf_model.tokenizer
|
||||
|
||||
# Some models need to explicitly pass the eos_token_id off the tokenizer
|
||||
# or processor for a good comparison;
|
||||
# currently assume processor/tokenizer agree on the EOS, and pull it off
|
||||
# the tokenizer if requested.
|
||||
hf_kwargs = {}
|
||||
if use_tokenizer_eos:
|
||||
hf_kwargs["eos_token_id"] = tokenizer.eos_token_id
|
||||
if stop_str:
|
||||
hf_kwargs["stop_strings"] = stop_str
|
||||
|
||||
for prompts, image_data, video_data, audio_data in inputs:
|
||||
mm_data = dict(images=image_data, videos=video_data, audios=audio_data)
|
||||
hf_kwargs_with_mm_data = hf_kwargs | mm_data
|
||||
hf_output = hf_model.generate_greedy_logprobs_limit(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
tokenizer=tokenizer,
|
||||
**hf_kwargs_with_mm_data,
|
||||
)
|
||||
hf_outputs_per_mm.append(hf_output)
|
||||
|
||||
# Apply output processing / sanitation to the vLLM and HF runner results
|
||||
hf_outputs_per_mm, vllm_outputs_per_mm = process_runner_outputs(
|
||||
model,
|
||||
first_runner_outputs=hf_outputs_per_mm,
|
||||
second_runner_outputs=vllm_outputs_per_mm,
|
||||
first_runner_processor=hf_output_post_proc,
|
||||
second_runner_processor=vllm_output_post_proc,
|
||||
)
|
||||
|
||||
for hf_outputs, vllm_outputs in zip(hf_outputs_per_mm, vllm_outputs_per_mm):
|
||||
# This is usually check_logprobs_close, but it's passed through to
|
||||
# allow things like check_outputs_equal where needed
|
||||
comparator(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
def process_runner_outputs(
|
||||
model,
|
||||
first_runner_outputs,
|
||||
second_runner_outputs,
|
||||
first_runner_processor=None,
|
||||
second_runner_processor=None,
|
||||
):
|
||||
"""Applies the runner processor(s) to the runner outputs, if any."""
|
||||
if first_runner_processor is not None:
|
||||
first_runner_outputs = process_outputs(
|
||||
first_runner_processor, model, first_runner_outputs
|
||||
)
|
||||
if second_runner_processor is not None:
|
||||
second_runner_outputs = process_outputs(
|
||||
second_runner_processor, model, second_runner_outputs
|
||||
)
|
||||
return first_runner_outputs, second_runner_outputs
|
||||
|
||||
|
||||
def process_outputs(output_processor, model, outputs_per_image):
|
||||
"""Applies a model specific post-processor function to a runner's output."""
|
||||
return [
|
||||
[output_processor(res, model) for res in outputs]
|
||||
for outputs in outputs_per_image
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,218 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Types for writing multimodal model tests."""
|
||||
|
||||
from collections.abc import Callable, Iterable
|
||||
from enum import Enum
|
||||
from pathlib import PosixPath
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
from pytest import MarkDecorator
|
||||
from transformers import AutoModelForCausalLM
|
||||
from transformers.models.auto.auto_factory import _BaseAutoModelClass
|
||||
|
||||
from vllm.config.model import RunnerOption
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .....conftest import (
|
||||
AUDIO_ASSETS,
|
||||
IMAGE_ASSETS,
|
||||
HfRunner,
|
||||
ImageAsset,
|
||||
ImageTestAssets,
|
||||
PromptAudioInput,
|
||||
PromptImageInput,
|
||||
PromptVideoInput,
|
||||
)
|
||||
from ....utils import check_logprobs_close
|
||||
|
||||
# meta image tag; will be replaced by the appropriate tag for the model
|
||||
TEST_IMG_PLACEHOLDER = "<vlm_image>"
|
||||
TEST_VIDEO_PLACEHOLDER = "<vlm_video>"
|
||||
TEST_AUDIO_PLACEHOLDER = "<lmm_audio>"
|
||||
|
||||
SINGLE_IMAGE_BASE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": f"{TEST_IMG_PLACEHOLDER}What's the content of the image?",
|
||||
"cherry_blossom": f"{TEST_IMG_PLACEHOLDER}What is the season?",
|
||||
}
|
||||
)
|
||||
SINGLE_AUDIO_BASE_PROMPT = AUDIO_ASSETS.prompts(
|
||||
{
|
||||
"mary_had_lamb": f"{TEST_AUDIO_PLACEHOLDER}Transcribe this audio into English.", # noqa: E501
|
||||
"winning_call": f"{TEST_AUDIO_PLACEHOLDER}What is happening in this audio clip?", # noqa: E501
|
||||
}
|
||||
)
|
||||
|
||||
MULTI_IMAGE_BASE_PROMPT = f"Image-1: {TEST_IMG_PLACEHOLDER}Image-2: {TEST_IMG_PLACEHOLDER}Describe the two images in detail.\n" # noqa: E501
|
||||
VIDEO_BASE_PROMPT = f"{TEST_VIDEO_PLACEHOLDER}Why is this video funny?"
|
||||
|
||||
|
||||
IMAGE_SIZE_FACTORS = [(1.0,), (1.0, 1.0, 1.0), (0.25, 0.5, 1.0)]
|
||||
EMBEDDING_SIZE_FACTORS = [(1.0,), (1.0, 1.0, 1.0)]
|
||||
RunnerOutput = tuple[list[int], str, SampleLogprobs | None]
|
||||
|
||||
|
||||
class PromptWithMultiModalInput(NamedTuple):
|
||||
"""Holds the multimodal input for a single test case."""
|
||||
|
||||
prompts: list[str]
|
||||
image_data: PromptImageInput | None = None
|
||||
video_data: PromptVideoInput | None = None
|
||||
audio_data: PromptAudioInput | None = None
|
||||
|
||||
|
||||
class VLMTestType(Enum):
|
||||
IMAGE = 1
|
||||
MULTI_IMAGE = 2
|
||||
EMBEDDING = 3
|
||||
VIDEO = 4
|
||||
AUDIO = 5
|
||||
CUSTOM_INPUTS = 6
|
||||
|
||||
|
||||
class SizeType(Enum):
|
||||
SIZE_FACTOR = 1
|
||||
FIXED_SIZE = 2
|
||||
|
||||
|
||||
class CustomTestOptions(NamedTuple):
|
||||
inputs: list[PromptWithMultiModalInput]
|
||||
limit_mm_per_prompt: dict[str, int]
|
||||
|
||||
|
||||
class ImageSizeWrapper(NamedTuple):
|
||||
type: SizeType
|
||||
# A size factor is a wrapper of 0+ floats,
|
||||
# while a fixed size contains an iterable of integer pairs
|
||||
data: Iterable[float] | Iterable[tuple[int, int]]
|
||||
|
||||
|
||||
class VLMTestInfo(NamedTuple):
|
||||
"""Holds the configuration for 1+ tests for one model architecture."""
|
||||
|
||||
models: list[str]
|
||||
test_type: VLMTestType | Iterable[VLMTestType]
|
||||
|
||||
# Should be None only if this is a CUSTOM_INPUTS test
|
||||
prompt_formatter: Callable[[str], str] | None = None
|
||||
img_idx_to_prompt: Callable[[int], str] = lambda idx: "<image>\n"
|
||||
video_idx_to_prompt: Callable[[int], str] = lambda idx: "<video>\n"
|
||||
audio_idx_to_prompt: Callable[[int], str] = lambda idx: "<audio>\n"
|
||||
|
||||
# Most models work on the single / multi-image prompts above, but in some
|
||||
# cases the log prob check fails, e.g., for paligemma. We allow passing
|
||||
# an override for the single image prompts / multi-image prompt for this
|
||||
# reason.
|
||||
single_image_prompts: Iterable[str] = SINGLE_IMAGE_BASE_PROMPTS
|
||||
multi_image_prompt: str = MULTI_IMAGE_BASE_PROMPT
|
||||
|
||||
# Function for converting ImageAssets to image embeddings;
|
||||
# We need to define this explicitly for embedding tests
|
||||
convert_assets_to_embeddings: (
|
||||
Callable[[ImageTestAssets], list[torch.Tensor]] | None
|
||||
) = None
|
||||
|
||||
# Exposed options for vLLM runner; we change these in a several tests,
|
||||
# but the defaults are derived from VllmRunner & the engine defaults
|
||||
# These settings are chosen to avoid OOMs when running in the CI
|
||||
enforce_eager: bool = True
|
||||
max_model_len: int = 1024
|
||||
max_num_seqs: int = 256
|
||||
runner: RunnerOption = "auto"
|
||||
tensor_parallel_size: int = 1
|
||||
vllm_runner_kwargs: dict[str, Any] | None = None
|
||||
|
||||
# Optional callable which gets a list of token IDs from the model tokenizer
|
||||
get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None = None
|
||||
# Optional list of strings to stop generation, useful when stop tokens are
|
||||
# not special tokens in the tokenizer
|
||||
stop_str: list[str] | None = None
|
||||
|
||||
# Exposed options for HF runner
|
||||
hf_model_kwargs: dict[str, Any] | None = None
|
||||
hf_processor: Callable[[str], Any] | None = None
|
||||
# Indicates we should explicitly pass the EOS from the tokenizer
|
||||
use_tokenizer_eos: bool = False
|
||||
auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM
|
||||
patch_hf_runner: Callable[[HfRunner], HfRunner] | None = None
|
||||
|
||||
# Post processors that if defined, will run oun the outputs of the
|
||||
# vLLM and HF runner, respectively (useful for sanitization, etc).
|
||||
vllm_output_post_proc: Callable[[RunnerOutput, str], Any] | None = None
|
||||
hf_output_post_proc: Callable[[RunnerOutput, str], Any] | None = None
|
||||
|
||||
# Consumes the output of the callables above and checks if they're equal
|
||||
comparator: Callable[..., None] = check_logprobs_close
|
||||
|
||||
# Default expandable params per test; these defaults can be overridden in
|
||||
# instances of this object; the complete set of test cases for the model
|
||||
# is all combinations of .models + all fields below
|
||||
max_tokens: int = 128
|
||||
num_logprobs: int = 5
|
||||
dtype: str = "auto"
|
||||
distributed_executor_backend: str | None = None
|
||||
# Only expanded in video tests
|
||||
num_video_frames: int | tuple[int] = 16
|
||||
needs_video_metadata: bool = False
|
||||
|
||||
# Fixed image sizes / image size factors; most tests use image_size_factors.
|
||||
# Each inner iterable defines one request batch. All batches are run against
|
||||
# the same model instance.
|
||||
image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS
|
||||
image_sizes: Iterable[Iterable[tuple[int, int]]] | None = None
|
||||
|
||||
# Hack for updating a prompt to take into a local path; currently only used
|
||||
# for Qwen-VL, which requires encoding the image path / url into the prompt
|
||||
# for HF runner
|
||||
prompt_path_encoder: (
|
||||
Callable[[PosixPath, str, list[ImageAsset] | ImageTestAssets], str] | None
|
||||
) = None # noqa: E501
|
||||
|
||||
# Allows configuring a test to run with custom inputs
|
||||
custom_test_opts: list[CustomTestOptions] | None = None
|
||||
|
||||
marks: list[MarkDecorator] | None = None
|
||||
|
||||
def get_non_parametrized_runner_kwargs(self):
|
||||
"""Returns a dictionary of expandable kwargs for items that are used
|
||||
in all test types, which are NOT used when creating the parametrized
|
||||
test cases.
|
||||
"""
|
||||
return {
|
||||
"enforce_eager": self.enforce_eager,
|
||||
"max_model_len": self.max_model_len,
|
||||
"max_num_seqs": self.max_num_seqs,
|
||||
"runner": self.runner,
|
||||
"tensor_parallel_size": self.tensor_parallel_size,
|
||||
"vllm_runner_kwargs": self.vllm_runner_kwargs,
|
||||
"hf_output_post_proc": self.hf_output_post_proc,
|
||||
"vllm_output_post_proc": self.vllm_output_post_proc,
|
||||
"auto_cls": self.auto_cls,
|
||||
"use_tokenizer_eos": self.use_tokenizer_eos,
|
||||
"comparator": self.comparator,
|
||||
"get_stop_token_ids": self.get_stop_token_ids,
|
||||
"hf_model_kwargs": self.hf_model_kwargs,
|
||||
"hf_processor": self.hf_processor,
|
||||
"stop_str": self.stop_str,
|
||||
"patch_hf_runner": self.patch_hf_runner,
|
||||
}
|
||||
|
||||
|
||||
class ExpandableVLMTestArgs(NamedTuple):
|
||||
"""The expanded kwargs which correspond to a single test case."""
|
||||
|
||||
model: str
|
||||
max_tokens: int
|
||||
num_logprobs: int
|
||||
dtype: str
|
||||
distributed_executor_backend: str | None
|
||||
# Sizes are used for everything except audio and custom input tests.
|
||||
size_wrappers: tuple[ImageSizeWrapper, ...] = ()
|
||||
# Video only
|
||||
num_video_frames: int | None = None
|
||||
needs_video_metadata: bool = False
|
||||
# Custom inputs only
|
||||
custom_test_opts: CustomTestOptions | None = None
|
||||
@@ -1,40 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import GenerateModelInfo
|
||||
|
||||
from .ppl_utils import vqa_ppl_test
|
||||
|
||||
MODELS = [
|
||||
(GenerateModelInfo("zai-org/GLM-OCR", hf_ppl=13.885052680969238), 28),
|
||||
]
|
||||
|
||||
|
||||
def _pixel_size(factor: int):
|
||||
# Match max image token size with qwen's PPL test
|
||||
return {
|
||||
"size": {
|
||||
"shortest_edge": 2 * factor**2,
|
||||
"longest_edge": 2 * 1280 * factor**2,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info,factor", MODELS)
|
||||
@pytest.mark.parametrize("mm_device_do_normalize", [True, False])
|
||||
def test_ppl(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info: GenerateModelInfo,
|
||||
factor: int,
|
||||
mm_device_do_normalize: bool,
|
||||
):
|
||||
vqa_ppl_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},
|
||||
mm_processor_kwargs=_pixel_size(factor),
|
||||
)
|
||||
@@ -1,140 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for Qwen2.5-Omni and Qwen3-Omni audio-in-video processor
|
||||
caching.
|
||||
|
||||
Tests the use_audio_in_video feature where audio is extracted from video and
|
||||
processed together with video frames in an interleaved manner.
|
||||
|
||||
Regression test: when use_audio_in_video=True and the multimodal processor
|
||||
cache is warm, the second request goes through MultiModalProcessorSenderCache
|
||||
which sets mm_kwargs["video"] items to None on a cache hit. The processor
|
||||
must still detect use_audio_in_video=True (via token-count heuristic) and
|
||||
produce the same prompt_token_ids as the first (cache-miss) request.
|
||||
|
||||
Without the fix the cache-hit path left use_audio_in_video=False, causing
|
||||
audio placeholder tokens to be inserted separately instead of being derived
|
||||
from the interleaved video placeholders – yielding a different (wrong) token
|
||||
sequence on every subsequent request for the same video.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import LruKeyReplicatedSenderCache
|
||||
|
||||
from ....multimodal.utils import random_audio, random_video
|
||||
from ...utils import build_model_context
|
||||
|
||||
MODELS = [
|
||||
"Qwen/Qwen2.5-Omni-3B",
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
]
|
||||
|
||||
|
||||
def create_mm_data(num_videos: int) -> dict[str, list]:
|
||||
# Small video (8 frames, 64×64) and ~0.5 s of audio at 16 kHz so the test
|
||||
# stays fast even without a GPU.
|
||||
mm_data = dict[str, list](video=[], audio=[])
|
||||
for i in range(num_videos):
|
||||
rng = np.random.RandomState(i)
|
||||
video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
|
||||
audio, sr = random_audio(rng, min_len=8000, max_len=8001, sr=16000)
|
||||
mm_data["video"].append(video)
|
||||
mm_data["audio"].append((audio, sr))
|
||||
return mm_data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", MODELS)
|
||||
@pytest.mark.parametrize("num_videos", [1, 2])
|
||||
def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> None:
|
||||
"""Regression test for https://github.com/vllm-project/vllm/pull/36800.
|
||||
|
||||
MultiModalProcessorSenderCache.get_and_update_item returns (None, updates)
|
||||
on a cache hit, so mm_kwargs["video"] items become None on the second call.
|
||||
The Qwen processor override of _maybe_apply_prompt_updates must detect
|
||||
use_audio_in_video=True via token-count heuristics and re-derive the audio
|
||||
placeholders correctly.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": num_videos, "image": 0, "video": num_videos},
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Sender cache: on a cache hit returns (None, prompt_updates) for each
|
||||
# item, setting mm_kwargs["video"] = [None] – the exact condition that
|
||||
# triggered the original bug.
|
||||
sender_cache = LruKeyReplicatedSenderCache(ctx.model_config)
|
||||
|
||||
video_token_id = processor.info.get_hf_config().video_token_id
|
||||
|
||||
mm_data = create_mm_data(num_videos)
|
||||
hf_processor_mm_kwargs = {"use_audio_in_video": True}
|
||||
|
||||
def run(processor, cache):
|
||||
return processor(
|
||||
[video_token_id] * num_videos,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
cache=cache,
|
||||
)["prompt_token_ids"]
|
||||
|
||||
baseline_ids = run(processor, cache=None)
|
||||
|
||||
# First call on the sender-cache processor: cache miss.
|
||||
# mm_kwargs["video"] items are real tensors; use_audio_in_video is
|
||||
# detected normally from the item data.
|
||||
first_ids = run(processor, cache=sender_cache)
|
||||
assert first_ids == baseline_ids, (
|
||||
"Cache-miss call produced different prompt_token_ids than baseline.\n"
|
||||
f" baseline : {baseline_ids}\n"
|
||||
f" cache-miss: {first_ids}"
|
||||
)
|
||||
|
||||
# Second call on the sender-cache processor: cache hit.
|
||||
# MultiModalProcessorSenderCache.get_and_update_item returns (None, …),
|
||||
# so mm_kwargs["video"] = [None]. Before the fix, use_audio_in_video was
|
||||
# not detected, yielding wrong token ids.
|
||||
second_ids = run(processor, cache=sender_cache)
|
||||
assert second_ids == baseline_ids, (
|
||||
"Cache-hit call produced different prompt_token_ids than baseline.\n"
|
||||
"This is the regression introduced when use_audio_in_video detection\n"
|
||||
"fails for None mm_kwargs items on a cache hit.\n"
|
||||
f" baseline : {baseline_ids}\n"
|
||||
f" cache-hit: {second_ids}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", MODELS)
|
||||
def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
|
||||
"""A video with no audio track, combined with `use_audio_in_video=True`,
|
||||
must raise a clear `ValueError` naming the mismatch, rather than an
|
||||
opaque `StopIteration`-derived error from the underlying HF processor.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
video_token_id = processor.info.get_hf_config().video_token_id
|
||||
|
||||
rng = np.random.RandomState(0)
|
||||
video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
|
||||
|
||||
# No "audio" key at all in this request's data: this is the "video has
|
||||
# no audio track" case. (The audio limit above is 1, not 0, so that
|
||||
# `mm_counts["audio"]` is still declared for this request; a 0 limit
|
||||
# would make vLLM omit the "audio" key from `mm_counts` entirely, which
|
||||
# trips an unrelated assertion in `_apply_hf_processor_main`.)
|
||||
mm_data = {"video": [video]}
|
||||
|
||||
with pytest.raises(ValueError, match="doesn't have audio track"):
|
||||
processor(
|
||||
[video_token_id],
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={"use_audio_in_video": True},
|
||||
)
|
||||
@@ -1,113 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
MODEL_ID = "nvidia/Cosmos3-Edge"
|
||||
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
LOCAL_MODEL_PATH = os.getenv("COSMOS3_EDGE_MODEL_PATH")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def processor():
|
||||
if LOCAL_MODEL_PATH:
|
||||
model_config = ModelConfig(
|
||||
LOCAL_MODEL_PATH,
|
||||
tokenizer=LOCAL_MODEL_PATH,
|
||||
max_model_len=4096,
|
||||
limit_mm_per_prompt={"image": 2, "video": 1},
|
||||
)
|
||||
else:
|
||||
ctx = build_model_context(
|
||||
MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 2, "video": 1},
|
||||
)
|
||||
model_config = ctx.model_config
|
||||
|
||||
return MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
|
||||
|
||||
def _assert_image_outputs(processor, processed, num_images: int) -> None:
|
||||
mm_data = processed["mm_kwargs"].get_data()
|
||||
grid_thw = mm_data["image_grid_thw"]
|
||||
pixel_values = mm_data["pixel_values"]
|
||||
|
||||
assert grid_thw.shape == (num_images, 3)
|
||||
assert pixel_values.shape[0] == int(grid_thw.prod(dim=-1).sum())
|
||||
|
||||
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
|
||||
expected_tokens = (grid_thw.prod(dim=-1) // merge_size**2).tolist()
|
||||
image_placeholders = processed["mm_placeholders"]["image"]
|
||||
|
||||
assert len(image_placeholders) == num_images
|
||||
assert [placeholder.length for placeholder in image_placeholders] == (
|
||||
expected_tokens
|
||||
)
|
||||
|
||||
image_token_id = processor.info.get_hf_processor().image_token_id
|
||||
assert processed["prompt_token_ids"].count(image_token_id) == sum(expected_tokens)
|
||||
|
||||
|
||||
def _assert_video_outputs(processor, processed) -> None:
|
||||
mm_data = processed["mm_kwargs"].get_data()
|
||||
grid_thw = mm_data["video_grid_thw"]
|
||||
pixel_values = mm_data["pixel_values_videos"]
|
||||
|
||||
assert grid_thw.shape == (1, 3)
|
||||
assert pixel_values.shape[0] == int(grid_thw.prod())
|
||||
assert len(processed["mm_placeholders"]["video"]) == 1
|
||||
|
||||
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
|
||||
expected_tokens = int(grid_thw.prod()) // merge_size**2
|
||||
video_token_id = processor.info.get_hf_config().video_token_id
|
||||
prompt_token_ids = processed["prompt_token_ids"]
|
||||
assert prompt_token_ids.count(video_token_id) == expected_tokens
|
||||
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
expected_frame_wrappers = int(grid_thw[:, 0].sum())
|
||||
assert (
|
||||
prompt_token_ids.count(hf_processor.vision_start_token_id)
|
||||
== expected_frame_wrappers
|
||||
)
|
||||
assert (
|
||||
prompt_token_ids.count(hf_processor.vision_end_token_id)
|
||||
== expected_frame_wrappers
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_images", [1, 2])
|
||||
def test_process_images(
|
||||
processor,
|
||||
image_assets: ImageTestAssets,
|
||||
num_images: int,
|
||||
) -> None:
|
||||
images = [asset.pil_image for asset in image_assets[:num_images]]
|
||||
processed = processor(
|
||||
IMAGE_PLACEHOLDER * num_images,
|
||||
mm_items=processor.info.parse_mm_data({"image": images}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
_assert_image_outputs(processor, processed, num_images)
|
||||
|
||||
|
||||
def test_process_video(processor) -> None:
|
||||
video_asset = VideoAsset(name="baby_reading", num_frames=8)
|
||||
video = (video_asset.np_ndarrays, video_asset.metadata)
|
||||
processed = processor(
|
||||
VIDEO_PLACEHOLDER,
|
||||
mm_items=processor.info.parse_mm_data({"video": [video]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
_assert_video_outputs(processor, processed)
|
||||
@@ -1,181 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for H2OVL's multimodal preprocessing kwargs."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.image import rescale_image_size
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
def _get_expected_num_patches(
|
||||
config: PreTrainedConfig,
|
||||
image: Image.Image,
|
||||
num_imgs: int,
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
):
|
||||
from vllm.transformers_utils.processors.h2ovl import (
|
||||
calculate_h2ovl_targets,
|
||||
get_h2ovl_target_ratios,
|
||||
)
|
||||
|
||||
width, height = image.size
|
||||
|
||||
# Calculate the expected number of blocks
|
||||
if num_imgs == 1 and config.use_msac:
|
||||
# First pass
|
||||
blocks1, _, _, aspect_ratio = calculate_h2ovl_targets(
|
||||
orig_width=width,
|
||||
orig_height=height,
|
||||
target_ratios=get_h2ovl_target_ratios(
|
||||
min_num=1,
|
||||
max_num=max_num,
|
||||
prior_aspect_ratio=None,
|
||||
),
|
||||
image_size=config.vision_config.image_size,
|
||||
use_thumbnail=False, # Thumbnail is handled separately
|
||||
)
|
||||
|
||||
# Second pass
|
||||
blocks2, _, _, _ = calculate_h2ovl_targets(
|
||||
orig_width=width,
|
||||
orig_height=height,
|
||||
target_ratios=get_h2ovl_target_ratios(
|
||||
min_num=3,
|
||||
max_num=max_num,
|
||||
prior_aspect_ratio=aspect_ratio,
|
||||
),
|
||||
image_size=config.vision_config.image_size,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
|
||||
# Add thumbnail if use_thumbnail is True and total_blocks > 1
|
||||
if config.use_thumbnail:
|
||||
blocks1 += 1 if blocks1 > 1 else 0
|
||||
blocks2 += 1 if blocks2 > 1 else 0
|
||||
|
||||
# Total blocks is the sum of blocks from both passes minus
|
||||
# overlapping
|
||||
total_blocks = blocks1 + blocks2 - 1
|
||||
|
||||
return total_blocks
|
||||
|
||||
blocks, _, _, _ = calculate_h2ovl_targets(
|
||||
orig_width=width,
|
||||
orig_height=height,
|
||||
target_ratios=get_h2ovl_target_ratios(
|
||||
min_num,
|
||||
max_num,
|
||||
prior_aspect_ratio=None,
|
||||
),
|
||||
image_size=config.vision_config.image_size,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
expected_num_patches = blocks
|
||||
|
||||
if config.use_thumbnail and expected_num_patches > 1:
|
||||
expected_num_patches += 1
|
||||
|
||||
return expected_num_patches
|
||||
|
||||
|
||||
def _run_check(
|
||||
processor: BaseMultiModalProcessor,
|
||||
images: list[Image.Image],
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
mm_processor_kwargs: Mapping[str, object],
|
||||
):
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
config = processor.info.get_hf_config()
|
||||
|
||||
prompt = "<image>" * len(images)
|
||||
mm_data = {"image": images}
|
||||
|
||||
total_expected_num_patches = sum(
|
||||
_get_expected_num_patches(config, image, len(images), min_num, max_num)
|
||||
for image in images
|
||||
)
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
|
||||
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
|
||||
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
|
||||
|
||||
assert img_tok_count == 256 * total_expected_num_patches
|
||||
assert pixel_shape[0] == total_expected_num_patches
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
[
|
||||
"h2oai/h2ovl-mississippi-800m",
|
||||
"h2oai/h2ovl-mississippi-2b",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"size_factors",
|
||||
[
|
||||
# Single-scale
|
||||
[1.0],
|
||||
# Single-scale, batched
|
||||
[1.0, 1.0, 1.0],
|
||||
# Multi-scale
|
||||
[0.25, 0.5, 1.0],
|
||||
[4.0, 2.0, 1.0],
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("min_dynamic_patch", "max_dynamic_patch"),
|
||||
[(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
|
||||
)
|
||||
@pytest.mark.parametrize("dynamic_image_size", [True, False])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
model_id: str,
|
||||
image_assets: ImageTestAssets,
|
||||
size_factors: list[int],
|
||||
min_dynamic_patch: int,
|
||||
max_dynamic_patch: int,
|
||||
dynamic_image_size: bool | None,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
mm_processor_kwargs = {
|
||||
"min_dynamic_patch": min_dynamic_patch,
|
||||
"max_dynamic_patch": max_dynamic_patch,
|
||||
"dynamic_image_size": dynamic_image_size,
|
||||
}
|
||||
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": len(size_factors)},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
|
||||
min_num = min_dynamic_patch if dynamic_image_size else 1
|
||||
max_num = max_dynamic_patch if dynamic_image_size else 1
|
||||
|
||||
_run_check(
|
||||
processor,
|
||||
[rescale_image_size(image_assets[0].pil_image, f) for f in size_factors],
|
||||
min_num,
|
||||
max_num,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
@@ -1,76 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Idefics3's multimodal preprocessing kwargs."""
|
||||
|
||||
import pytest
|
||||
from transformers import Idefics3Config
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["HuggingFaceM4/Idefics3-8B-Llama3"])
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_toks_per_img"),
|
||||
[
|
||||
({"size": {"longest_edge": 364}}, 169),
|
||||
({"size": {"longest_edge": 728}}, 169 * (2**2 + 1)),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
expected_toks_per_img: int,
|
||||
num_imgs: int,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
"""Ensure Idefics3MultiModalProcessor handles num_crops properly."""
|
||||
# Same as the previous test - don't initialize mm_processor_kwargs
|
||||
# in this test and assume that the kwargs will be correctly expanded by
|
||||
# the partial when calling the custom input processor.
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
|
||||
# Build the image str / prompt based on the number of images we pass
|
||||
placeholders = (
|
||||
"<image>"
|
||||
if num_imgs == 1
|
||||
else "\n".join(f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
|
||||
)
|
||||
prompt = f"<|begin_of_text|>User:{placeholders}\n<end_of_utterance>\nAssistant:" # noqa: E501
|
||||
|
||||
# Build mm_data
|
||||
image_size = ctx.get_hf_config(Idefics3Config).vision_config.image_size
|
||||
dummy_image_size = (image_size * 4, image_size * 4)
|
||||
dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
|
||||
mm_data = {"image": [dummy_image] * num_imgs}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# Ensure the placeholders format are correct
|
||||
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
hf_processed_inputs = hf_processor(
|
||||
text=prompt,
|
||||
images=mm_data["image"],
|
||||
**processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs),
|
||||
)
|
||||
assert processed_inputs["prompt_token_ids"] == hf_processed_inputs["input_ids"][0]
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
image_token_id = ctx.get_hf_config().image_token_id
|
||||
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
|
||||
assert img_tok_count == expected_toks_per_img * num_imgs
|
||||
@@ -1,135 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for InternVL's multimodal preprocessing kwargs."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.image import rescale_image_size
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
def _get_expected_num_patches(
|
||||
config: PreTrainedConfig,
|
||||
image: Image.Image,
|
||||
num_imgs: int,
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
):
|
||||
from vllm.transformers_utils.processors.internvl import (
|
||||
calculate_internvl_targets,
|
||||
get_internvl_target_ratios,
|
||||
)
|
||||
|
||||
width, height = image.size
|
||||
|
||||
blocks, _, _ = calculate_internvl_targets(
|
||||
orig_width=width,
|
||||
orig_height=height,
|
||||
target_ratios=get_internvl_target_ratios(
|
||||
min_num,
|
||||
max_num,
|
||||
),
|
||||
image_size=config.vision_config.image_size,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
expected_num_patches = blocks
|
||||
|
||||
if config.use_thumbnail and expected_num_patches > 1:
|
||||
expected_num_patches += 1
|
||||
|
||||
return expected_num_patches
|
||||
|
||||
|
||||
def _run_check(
|
||||
processor: BaseMultiModalProcessor,
|
||||
images: list[Image.Image],
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
mm_processor_kwargs: Mapping[str, object],
|
||||
):
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
config = processor.info.get_hf_config()
|
||||
|
||||
prompt = "<image>" * len(images)
|
||||
mm_data = {"image": images}
|
||||
|
||||
total_expected_num_patches = sum(
|
||||
_get_expected_num_patches(config, image, len(images), min_num, max_num)
|
||||
for image in images
|
||||
)
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
|
||||
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
|
||||
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
|
||||
|
||||
assert img_tok_count == 256 * total_expected_num_patches
|
||||
assert pixel_shape[0] == total_expected_num_patches
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["OpenGVLab/InternVL2-2B"])
|
||||
@pytest.mark.parametrize(
|
||||
"size_factors",
|
||||
[
|
||||
# Single-scale
|
||||
[1.0],
|
||||
# Single-scale, batched
|
||||
[1.0, 1.0, 1.0],
|
||||
# Multi-scale
|
||||
[0.25, 0.5, 1.0],
|
||||
[4.0, 2.0, 1.0],
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("min_dynamic_patch", "max_dynamic_patch"),
|
||||
[(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
|
||||
)
|
||||
@pytest.mark.parametrize("dynamic_image_size", [True, False])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
model_id: str,
|
||||
image_assets: ImageTestAssets,
|
||||
size_factors: list[int],
|
||||
min_dynamic_patch: int,
|
||||
max_dynamic_patch: int,
|
||||
dynamic_image_size: bool | None,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
mm_processor_kwargs = {
|
||||
"min_dynamic_patch": min_dynamic_patch,
|
||||
"max_dynamic_patch": max_dynamic_patch,
|
||||
"dynamic_image_size": dynamic_image_size,
|
||||
}
|
||||
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": len(size_factors)},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
|
||||
min_num = min_dynamic_patch if dynamic_image_size else 1
|
||||
max_num = max_dynamic_patch if dynamic_image_size else 1
|
||||
|
||||
_run_check(
|
||||
processor,
|
||||
[rescale_image_size(image_assets[0].pil_image, f) for f in size_factors],
|
||||
min_num,
|
||||
max_num,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
@@ -1,50 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for LightOnOCR precomputed image embeddings."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
|
||||
_MODEL_CONFIG_KWARGS = {"config_format": "hf"}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize("tensor_input", [False, True])
|
||||
@pytest.mark.parametrize("prompt_as_tokens", [False, True])
|
||||
def test_lightonocr_image_embeddings_preserve_lengths_and_values(
|
||||
tensor_input: bool, prompt_as_tokens: bool
|
||||
):
|
||||
"""Reserve each image's feature count and pass projected embeddings through."""
|
||||
ctx = build_model_context(
|
||||
_LIGHTON_MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
model_config_kwargs=_MODEL_CONFIG_KWARGS,
|
||||
)
|
||||
ctx.model_config.get_multimodal_config().enable_mm_embeds = True
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
hidden_size = ctx.model_config.get_hidden_size()
|
||||
feature_sizes = [3, 3] if tensor_input else [3, 7]
|
||||
embeddings = [
|
||||
torch.full((size, hidden_size), float(i))
|
||||
for i, size in enumerate(feature_sizes)
|
||||
]
|
||||
image_input = torch.stack(embeddings) if tensor_input else embeddings
|
||||
prompt = " ".join([hf_processor.image_token] * len(embeddings))
|
||||
if prompt_as_tokens:
|
||||
prompt = ctx.get_tokenizer().encode(prompt)
|
||||
|
||||
items = processor.info.parse_mm_data({"image": image_input})
|
||||
processed = processor(prompt, mm_items=items)
|
||||
|
||||
assert processed["prompt_token_ids"].count(hf_processor.image_token_id) == sum(
|
||||
feature_sizes
|
||||
)
|
||||
assert [p.length for p in processed["mm_placeholders"]["image"]] == feature_sizes
|
||||
for item, expected in zip(processed["mm_kwargs"]["image"], embeddings):
|
||||
torch.testing.assert_close(item.get_data()["image_embeds"], expected)
|
||||
@@ -1,89 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Llama4's multimodal preprocessing kwargs."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["meta-llama/Llama-4-Scout-17B-16E-Instruct"])
|
||||
@pytest.mark.parametrize("mm_processor_kwargs", [{}])
|
||||
@pytest.mark.parametrize("num_imgs", [1, 5])
|
||||
@pytest.mark.parametrize("mm_processor_cache_gb", [0, 4])
|
||||
@pytest.mark.parametrize("tokenized_prompt", [True, False])
|
||||
def test_processor_override(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict,
|
||||
num_imgs: int,
|
||||
mm_processor_cache_gb: int,
|
||||
tokenized_prompt: bool,
|
||||
):
|
||||
"""Ensure llama4 processor works properly."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
mm_processor_cache_gb=mm_processor_cache_gb,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
config = processor.info.get_hf_config()
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
prompt = (
|
||||
"<|begin_of_text|><|header_start|>user<|header_end|>"
|
||||
+ "<|image|>" * num_imgs
|
||||
+ "<|eot|><|header_start|>assistant<|header_end|>"
|
||||
)
|
||||
mm_data = {
|
||||
"image": [
|
||||
image_assets[(i % len(image_assets))].pil_image for i in range(num_imgs)
|
||||
]
|
||||
}
|
||||
if tokenized_prompt:
|
||||
prompt = tokenizer.encode(prompt)
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
mm_data = processed_inputs["mm_kwargs"].get_data()
|
||||
|
||||
# place holder replacements
|
||||
prompt_token_ids = processed_inputs["prompt_token_ids"]
|
||||
assert prompt_token_ids.count(config.boi_token_index) == num_imgs
|
||||
assert prompt_token_ids.count(config.eoi_token_index) == num_imgs
|
||||
assert prompt_token_ids.count(vocab[hf_processor.image_token]) == num_imgs
|
||||
aspect_ratios = mm_data["aspect_ratios"]
|
||||
num_x_separators = num_y_separators = 0
|
||||
for tiles_y, tiles_x in aspect_ratios:
|
||||
if tiles_x * tiles_y > 1:
|
||||
num_x_separators += (tiles_x - 1) * tiles_y
|
||||
num_y_separators += tiles_y
|
||||
assert prompt_token_ids.count(vocab[hf_processor.tile_token]) == num_x_separators
|
||||
assert (
|
||||
prompt_token_ids.count(vocab[hf_processor.tile_global_token])
|
||||
== num_y_separators
|
||||
)
|
||||
|
||||
# image token offsets
|
||||
img_locs = processed_inputs["mm_placeholders"].get("image", [])
|
||||
assert len(img_locs) == num_imgs
|
||||
assert [img_loc.offset for img_loc in img_locs] == [
|
||||
i for i, v in enumerate(prompt_token_ids) if v == config.boi_token_index
|
||||
]
|
||||
|
||||
# patch sizes and masks
|
||||
num_patches_per_chunk = processor.info.get_patch_per_chunk(config.vision_config)
|
||||
assert (
|
||||
prompt_token_ids.count(config.image_token_index)
|
||||
== sum(mm_data["patches_per_image"]) * num_patches_per_chunk
|
||||
)
|
||||
assert len(mm_data["pixel_values"]) == sum(mm_data["patches_per_image"])
|
||||
@@ -1,212 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import itertools
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from pqdm.threads import pqdm
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import ImageSize
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.tokenizers.hf import maybe_make_thread_pool
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
def _validate_image_max_tokens_one(
|
||||
processor: BaseMultiModalProcessor,
|
||||
max_tokens: int,
|
||||
failed_size_excs: list[tuple[ImageSize, Exception]],
|
||||
image_size: ImageSize,
|
||||
) -> None:
|
||||
info = processor.info
|
||||
feature_size = info.get_num_image_tokens(
|
||||
image_width=image_size.width, image_height=image_size.height
|
||||
)
|
||||
|
||||
try:
|
||||
assert feature_size <= max_tokens, f"{feature_size} <= {max_tokens}"
|
||||
except Exception as exc:
|
||||
failed_size_excs.append((image_size, exc))
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
"This test takes around 5 minutes to run. Comment this out to run it manually."
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
|
||||
def test_processor_max_tokens(model_id):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
info = processor.info
|
||||
|
||||
seen_aspect_ratios = set[float]()
|
||||
image_sizes = list[ImageSize]()
|
||||
|
||||
# The aspect ratio of the grid layout is between 1 and 2
|
||||
# NOTE: Assumes that feature size calculation is the same if we
|
||||
# swap the width and height of the image
|
||||
for w, h in itertools.product(range(32, 4096), repeat=2):
|
||||
aspect_ratio = w / h
|
||||
if 1 <= aspect_ratio <= 2 and aspect_ratio not in seen_aspect_ratios:
|
||||
image_sizes.append(ImageSize(w, h))
|
||||
seen_aspect_ratios.add(aspect_ratio)
|
||||
|
||||
failed_size_excs = list[tuple[ImageSize, Exception]]()
|
||||
|
||||
validate_one = partial(
|
||||
_validate_image_max_tokens_one,
|
||||
processor,
|
||||
info.get_max_image_tokens(), # type: ignore
|
||||
failed_size_excs,
|
||||
)
|
||||
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
|
||||
|
||||
if failed_size_excs:
|
||||
msg = "Found failing image sizes:" + "\n========\n".join(
|
||||
f"[{size}]\n{exc}" for size, exc in failed_size_excs
|
||||
)
|
||||
raise AssertionError(msg)
|
||||
|
||||
|
||||
def _validate_image_prompt_replacements_one(
|
||||
processor: BaseMultiModalProcessor,
|
||||
num_imgs: int,
|
||||
failed_size_excs: list[tuple[ImageSize, Exception]],
|
||||
image_size: ImageSize,
|
||||
) -> None:
|
||||
prompt = "<image>" * num_imgs
|
||||
image = Image.new("RGB", size=image_size)
|
||||
mm_data = {"image": [image] * num_imgs}
|
||||
|
||||
try:
|
||||
# The processor will throw an error if there is a mismatch
|
||||
# in the prompt replacements
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == num_imgs
|
||||
|
||||
first_placeholder = image_placeholders[0]
|
||||
|
||||
# NOTE: There is a BOS token
|
||||
assert first_placeholder.offset == 1
|
||||
assert (
|
||||
first_placeholder.length
|
||||
== (len(processed_inputs["prompt_token_ids"]) - 1) // num_imgs
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
failed_size_excs.append((image_size, exc))
|
||||
|
||||
|
||||
def _test_image_prompt_replacements(
|
||||
processor,
|
||||
*,
|
||||
num_imgs: int,
|
||||
image_sizes: list[ImageSize],
|
||||
) -> None:
|
||||
"""Ensure LlavaNextMultiModalProcessor
|
||||
handles prompt replacement properly for input images.
|
||||
"""
|
||||
failed_size_excs = list[tuple[ImageSize, Exception]]()
|
||||
|
||||
validate_one = partial(
|
||||
_validate_image_prompt_replacements_one,
|
||||
processor,
|
||||
num_imgs,
|
||||
failed_size_excs,
|
||||
)
|
||||
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
|
||||
|
||||
if failed_size_excs:
|
||||
msg = "Found failing image sizes:" + "\n========\n".join(
|
||||
f"[{size}]\n{exc}" for size, exc in failed_size_excs
|
||||
)
|
||||
raise AssertionError(msg)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
def test_processor_prompt_replacements_regression(model_id, num_imgs):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
image_ratios = [
|
||||
(171, 152),
|
||||
(184, 161),
|
||||
(198, 176),
|
||||
(333, 296),
|
||||
(369, 328),
|
||||
(488, 183),
|
||||
(2560, 1669),
|
||||
]
|
||||
image_sizes = [
|
||||
size for w, h in image_ratios for size in [ImageSize(w, h), ImageSize(h, w)]
|
||||
]
|
||||
|
||||
_test_image_prompt_replacements(
|
||||
processor,
|
||||
num_imgs=num_imgs,
|
||||
image_sizes=image_sizes,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
"This test takes around 2 hours to run. Comment this out to run it manually."
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
|
||||
@pytest.mark.parametrize("num_imgs", [1])
|
||||
def test_processor_prompt_replacements_all(model_id, num_imgs):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
seen_aspect_ratios = set[float]()
|
||||
image_sizes = list[ImageSize]()
|
||||
|
||||
# The aspect ratio of the grid layout is between 1 and 2
|
||||
# NOTE: Assumes that feature size calculation is the same if we
|
||||
# swap the width and height of the image
|
||||
for w, h in itertools.product(range(64, 1024), repeat=2):
|
||||
aspect_ratio = w / h
|
||||
if 1 <= aspect_ratio <= 2 and aspect_ratio not in seen_aspect_ratios:
|
||||
image_sizes.append(ImageSize(w, h))
|
||||
seen_aspect_ratios.add(aspect_ratio)
|
||||
|
||||
_test_image_prompt_replacements(
|
||||
processor,
|
||||
num_imgs=num_imgs,
|
||||
image_sizes=image_sizes,
|
||||
)
|
||||
@@ -1,211 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import itertools
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from pqdm.threads import pqdm
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import ImageSize
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.tokenizers.hf import maybe_make_thread_pool
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
def _validate_image_max_tokens_one(
|
||||
processor: BaseMultiModalProcessor,
|
||||
max_tokens: int,
|
||||
failed_size_excs: list[tuple[ImageSize, Exception]],
|
||||
image_size: ImageSize,
|
||||
) -> None:
|
||||
info = processor.info
|
||||
feature_size = info.get_num_image_tokens(
|
||||
image_width=image_size.width, image_height=image_size.height
|
||||
)
|
||||
|
||||
try:
|
||||
assert feature_size <= max_tokens, f"{feature_size} <= {max_tokens}"
|
||||
except Exception as exc:
|
||||
failed_size_excs.append((image_size, exc))
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
"This test takes around 5 minutes to run. Comment this out to run it manually."
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
||||
def test_processor_max_tokens(model_id):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
info = processor.info
|
||||
|
||||
seen_aspect_ratios = set[float]()
|
||||
image_sizes = list[ImageSize]()
|
||||
|
||||
# The aspect ratio of the grid layout is between 1 and 6
|
||||
# NOTE: Assumes that feature size calculation is the same if we
|
||||
# swap the width and height of the image
|
||||
for w, h in itertools.product(range(32, 4096), repeat=2):
|
||||
aspect_ratio = w / h
|
||||
if 1 <= aspect_ratio <= 6 and aspect_ratio not in seen_aspect_ratios:
|
||||
image_sizes.append(ImageSize(w, h))
|
||||
seen_aspect_ratios.add(aspect_ratio)
|
||||
|
||||
failed_size_excs = list[tuple[ImageSize, Exception]]()
|
||||
|
||||
validate_one = partial(
|
||||
_validate_image_max_tokens_one,
|
||||
processor,
|
||||
info.get_max_image_tokens(), # type: ignore
|
||||
failed_size_excs,
|
||||
)
|
||||
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
|
||||
|
||||
if failed_size_excs:
|
||||
msg = "Found failing image sizes:" + "\n========\n".join(
|
||||
f"[{size}]\n{exc}" for size, exc in failed_size_excs
|
||||
)
|
||||
raise AssertionError(msg)
|
||||
|
||||
|
||||
def _validate_image_prompt_replacements_one(
|
||||
processor: BaseMultiModalProcessor,
|
||||
num_imgs: int,
|
||||
failed_size_excs: list[tuple[ImageSize, Exception]],
|
||||
image_size: ImageSize,
|
||||
) -> None:
|
||||
prompt = "<image>" * num_imgs
|
||||
image = Image.new("RGB", size=image_size)
|
||||
mm_data = {"image": [image] * num_imgs}
|
||||
|
||||
try:
|
||||
# The processor will throw an error if there is a mismatch
|
||||
# in the prompt replacements
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == num_imgs
|
||||
|
||||
first_placeholder = image_placeholders[0]
|
||||
|
||||
assert first_placeholder.offset == 0
|
||||
assert (
|
||||
first_placeholder.length
|
||||
== len(processed_inputs["prompt_token_ids"]) // num_imgs
|
||||
)
|
||||
except Exception as exc:
|
||||
failed_size_excs.append((image_size, exc))
|
||||
|
||||
|
||||
def _test_image_prompt_replacements(
|
||||
processor,
|
||||
*,
|
||||
num_imgs: int,
|
||||
image_sizes: list[ImageSize],
|
||||
) -> None:
|
||||
"""Ensure LlavaOnevisionMultiModalProcessor
|
||||
handles prompt replacement properly for input images.
|
||||
"""
|
||||
failed_size_excs = list[tuple[ImageSize, Exception]]()
|
||||
|
||||
validate_one = partial(
|
||||
_validate_image_prompt_replacements_one,
|
||||
processor,
|
||||
num_imgs,
|
||||
failed_size_excs,
|
||||
)
|
||||
pqdm(image_sizes, validate_one, n_jobs=8, desc="Validating image sizes")
|
||||
|
||||
if failed_size_excs:
|
||||
msg = "Found failing image sizes:" + "\n========\n".join(
|
||||
f"[{size}]\n{exc}" for size, exc in failed_size_excs
|
||||
)
|
||||
raise AssertionError(msg)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
def test_processor_prompt_replacements_regression(model_id, num_imgs):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
|
||||
# Avoid tokenizer already borrowed error
|
||||
maybe_make_thread_pool(ctx.tokenizer)
|
||||
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
image_ratios = [
|
||||
(171, 152),
|
||||
(184, 161),
|
||||
(198, 176),
|
||||
(333, 296),
|
||||
(369, 328),
|
||||
(488, 183),
|
||||
(2560, 1669),
|
||||
]
|
||||
image_sizes = [
|
||||
size for w, h in image_ratios for size in [ImageSize(w, h), ImageSize(h, w)]
|
||||
]
|
||||
|
||||
_test_image_prompt_replacements(
|
||||
processor,
|
||||
num_imgs=num_imgs,
|
||||
image_sizes=image_sizes,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
"This test takes around 2 hours to run. Comment this out to run it manually."
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
||||
@pytest.mark.parametrize("num_imgs", [1])
|
||||
def test_processor_prompt_replacements_all(model_id, num_imgs):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
seen_aspect_ratios = set[float]()
|
||||
image_sizes = list[ImageSize]()
|
||||
|
||||
# The aspect ratio of the grid layout is between 1 and 6
|
||||
# NOTE: Assumes that feature size calculation is the same if we
|
||||
# swap the width and height of the image
|
||||
for w, h in itertools.product(range(64, 1024), repeat=2):
|
||||
aspect_ratio = w / h
|
||||
if 1 <= aspect_ratio <= 6 and aspect_ratio not in seen_aspect_ratios:
|
||||
image_sizes.append(ImageSize(w, h))
|
||||
seen_aspect_ratios.add(aspect_ratio)
|
||||
|
||||
_test_image_prompt_replacements(
|
||||
processor,
|
||||
num_imgs=num_imgs,
|
||||
image_sizes=image_sizes,
|
||||
)
|
||||
@@ -1,90 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for MiniCPMV's multimodal preprocessing."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import ImageSize
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
|
||||
def test_get_hf_processor_for_same_model_different_kwargs(model_id: str):
|
||||
"""Calls with different kwargs must not reuse stale processor instances."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
info = processor.info
|
||||
processor_1 = info.get_hf_processor(max_slice_nums=1)
|
||||
processor_2 = info.get_hf_processor(max_slice_nums=2)
|
||||
assert processor_1.image_processor.max_slice_nums == 1
|
||||
assert processor_2.image_processor.max_slice_nums == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_ids",
|
||||
[
|
||||
("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-V-4"),
|
||||
("openbmb/MiniCPM-Llama3-V-2_5", "openbmb/MiniCPM-o-2_6"),
|
||||
],
|
||||
)
|
||||
def test_image_processor_for_different_models(model_ids):
|
||||
first_model_id, second_model_id = model_ids
|
||||
|
||||
first_ctx = build_model_context(
|
||||
first_model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
first_processor = MULTIMODAL_REGISTRY.create_processor(first_ctx.model_config)
|
||||
first_image_processor = first_processor.info.get_image_processor()
|
||||
|
||||
second_ctx = build_model_context(
|
||||
second_model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
second_processor = MULTIMODAL_REGISTRY.create_processor(second_ctx.model_config)
|
||||
second_image_processor = second_processor.info.get_image_processor()
|
||||
|
||||
second_processor.info.get_sliced_grid(ImageSize(width=128, height=128))
|
||||
|
||||
assert type(first_image_processor) is not type(second_image_processor)
|
||||
assert (
|
||||
type(first_image_processor).__module__
|
||||
!= type(second_image_processor).__module__
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["openbmb/MiniCPM-V-4"])
|
||||
def test_prompt_has_dif_BPE_boundaries_in_context(model_id: str):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
tokenizer = ctx.get_tokenizer()
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "(<image>./</image>)\nWhat is in this image?"}
|
||||
]
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
image = np.zeros((768, 1024, 3), dtype=np.uint8)
|
||||
|
||||
mm_items = processor.info.parse_mm_data({"image": [image]})
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
image_placeholders = processed["mm_placeholders"].get("image", [])
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length > 0
|
||||
@@ -1,138 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for MiniMax-M3 VL ``max_long_side_pixel`` resize support.
|
||||
|
||||
These exercise the vendored processor directly (no checkpoint / GPU needed), so
|
||||
they validate the long-side resize spec and the resulting prompt-token counts
|
||||
deterministically.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.transformers_utils.processors.minimax_m3 import (
|
||||
IMAGE_MAX_TOTAL_PIXELS,
|
||||
MIN_SHORT_SIDE_PIXEL,
|
||||
VIDEO_MAX_TOTAL_PIXELS,
|
||||
MiniMaxM3VLImageProcessor,
|
||||
MiniMaxM3VLVideoProcessor,
|
||||
smart_resize,
|
||||
)
|
||||
|
||||
# Long sides are multiples of patch_size*merge_size (28) so the rounding is
|
||||
# exact and the expected token counts are unambiguous.
|
||||
LONG_SIDES = [252, 504, 1008]
|
||||
MERGE2 = 2**2 # merge_size ** 2
|
||||
|
||||
|
||||
def _image_tokens(grid_thw) -> int:
|
||||
g = list(grid_thw)
|
||||
return int(g[0] * g[1] * g[2]) // MERGE2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# smart_resize: the long-side spec (a) shrink / (b) enlarge / (c) hard cap
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_smart_resize_long_side_shrink():
|
||||
# (a) long side exceeds the cap -> shrink so the long side equals the cap.
|
||||
h, w = smart_resize(
|
||||
2048, 1024, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
|
||||
)
|
||||
assert max(h, w) == 1008
|
||||
assert (h, w) == (1008, 504) # aspect ratio preserved
|
||||
|
||||
|
||||
def test_smart_resize_short_side_enlarge():
|
||||
# (b) long side within the cap but short side below the floor -> enlarge so
|
||||
# the short side reaches min_short_side_pixel.
|
||||
h, w = smart_resize(
|
||||
200, 40, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
|
||||
)
|
||||
assert min(h, w) == MIN_SHORT_SIDE_PIXEL # 112
|
||||
|
||||
|
||||
def test_smart_resize_total_pixels_raises():
|
||||
# (c) still over the area cap after resizing -> raise instead of inferring.
|
||||
with pytest.raises(ValueError, match="max_total_pixels"):
|
||||
smart_resize(
|
||||
5000,
|
||||
5000,
|
||||
factor=28,
|
||||
max_long_side_pixel=4000,
|
||||
max_total_pixels=IMAGE_MAX_TOTAL_PIXELS,
|
||||
)
|
||||
|
||||
|
||||
def test_smart_resize_backward_compatible_area_bound():
|
||||
# Without max_long_side_pixel the original Qwen-style area bound is used.
|
||||
assert smart_resize(2048, 2048, factor=28, max_pixels=451584) == (672, 672)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Image processor: monotonic prompt-token counts for 252 < 504 < 1008
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_image_tokens_increase_with_max_long_side_pixel():
|
||||
proc = MiniMaxM3VLImageProcessor()
|
||||
counts = []
|
||||
for long_side in LONG_SIDES:
|
||||
patches = proc.get_number_of_image_patches(
|
||||
2048, 2048, images_kwargs={"max_long_side_pixel": long_side}
|
||||
)
|
||||
counts.append(patches // MERGE2)
|
||||
|
||||
assert counts == [81, 324, 1296]
|
||||
assert counts[0] < counts[1] < counts[2]
|
||||
|
||||
|
||||
def test_image_processor_defaults_match_spec():
|
||||
proc = MiniMaxM3VLImageProcessor()
|
||||
assert proc.max_long_side_pixel is None # opt-in
|
||||
assert proc.min_short_side_pixel == MIN_SHORT_SIDE_PIXEL
|
||||
assert proc.max_total_pixels == IMAGE_MAX_TOTAL_PIXELS
|
||||
|
||||
|
||||
def test_image_preprocess_pipeline_monotonic():
|
||||
proc = MiniMaxM3VLImageProcessor()
|
||||
image = torch.randint(0, 255, (3, 2048, 2048), dtype=torch.uint8)
|
||||
counts = []
|
||||
for long_side in LONG_SIDES:
|
||||
out = proc.preprocess(
|
||||
[image],
|
||||
do_resize=True,
|
||||
max_long_side_pixel=long_side,
|
||||
return_tensors="pt",
|
||||
)
|
||||
counts.append(_image_tokens(out["image_grid_thw"][0]))
|
||||
assert counts == [81, 324, 1296]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Video processor: same monotonic behavior + volumetric (w*h*frames) cap
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_video_tokens_increase_with_max_long_side_pixel():
|
||||
proc = MiniMaxM3VLVideoProcessor()
|
||||
assert proc.max_total_pixels == VIDEO_MAX_TOTAL_PIXELS
|
||||
video = torch.randint(0, 255, (4, 3, 2048, 2048), dtype=torch.uint8)
|
||||
counts = []
|
||||
for long_side in LONG_SIDES:
|
||||
out = proc.preprocess(
|
||||
videos=[video],
|
||||
do_resize=True,
|
||||
max_long_side_pixel=long_side,
|
||||
return_tensors="pt",
|
||||
)
|
||||
counts.append(_image_tokens(out["video_grid_thw"][0]))
|
||||
assert counts[0] < counts[1] < counts[2]
|
||||
|
||||
|
||||
def test_video_volumetric_cap_raises():
|
||||
proc = MiniMaxM3VLVideoProcessor()
|
||||
# 400 frames at a 1008-long-side square: 1008*1008*400 >> 301,056,000.
|
||||
video = torch.randint(0, 255, (400, 3, 2048, 2048), dtype=torch.uint8)
|
||||
with pytest.raises(ValueError, match="max_total_pixels"):
|
||||
proc.preprocess(
|
||||
videos=[video],
|
||||
do_resize=True,
|
||||
max_long_side_pixel=1008,
|
||||
return_tensors="pt",
|
||||
)
|
||||
@@ -1,53 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for mllama's multimodal preprocessing and profiling."""
|
||||
|
||||
import pytest
|
||||
from torch import prod
|
||||
from transformers import Llama4Config
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["meta-llama/Llama-Guard-4-12B"])
|
||||
@pytest.mark.parametrize("max_model_len", [4096, 8192, 25600, 131072])
|
||||
def test_profiling(model_id: str, max_model_len: int):
|
||||
model_config_kwargs = {
|
||||
"max_model_len": max_model_len,
|
||||
}
|
||||
mm_counts = {"image": 1}
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
model_config_kwargs=model_config_kwargs,
|
||||
limit_mm_per_prompt=mm_counts,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
mm_inputs = processor.get_dummy_mm_inputs(mm_counts)
|
||||
|
||||
hf_config = ctx.get_hf_config(Llama4Config)
|
||||
image_size = hf_config.vision_config.image_size
|
||||
patch_size = hf_config.vision_config.patch_size
|
||||
downsample_ratio = int(
|
||||
round(1.0 / (hf_config.vision_config.pixel_shuffle_ratio**2))
|
||||
)
|
||||
tokens_per_patch = ((image_size // patch_size) ** 2) // downsample_ratio
|
||||
|
||||
mm_data = mm_inputs["mm_kwargs"].get_data()
|
||||
chunks_per_image = prod(mm_data["patches_per_image"])
|
||||
total_num_patches = chunks_per_image * tokens_per_patch
|
||||
num_tiles = (
|
||||
mm_data["aspect_ratios"][0][0] * mm_data["aspect_ratios"][0][1]
|
||||
) # x-y separator tokens
|
||||
total_tokens = (
|
||||
total_num_patches.item() + num_tiles.item() + 3
|
||||
) # image start, image, image end
|
||||
|
||||
assert total_num_patches == sum(
|
||||
item.get_num_embeds() for item in mm_inputs["mm_placeholders"]["image"]
|
||||
)
|
||||
assert total_tokens == sum(
|
||||
placeholder.length for placeholder in mm_inputs["mm_placeholders"]["image"]
|
||||
)
|
||||
@@ -1,553 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Moondream3 multimodal processing.
|
||||
|
||||
Includes:
|
||||
- Processor creation and application tests
|
||||
- Image tokenization and placeholder expansion tests
|
||||
- Tiling and cropping logic tests (CPU-based)
|
||||
- Pixel normalization tests
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
MOONDREAM3_MODEL_ID = "moondream/moondream3-preview"
|
||||
# Expected multimodal prefix: BOS + 729 image tokens.
|
||||
EXPECTED_IMAGE_TOKENS = 730
|
||||
# Vision encoder constants
|
||||
CROP_SIZE = 378
|
||||
PATCH_SIZE = 14
|
||||
MAX_CROPS = 12
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_creation(model_id: str):
|
||||
"""Test that Moondream3 processor can be created."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
assert processor is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_apply(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that Moondream3 processor can process inputs.
|
||||
|
||||
NOTE: The prompt includes the leading BOS token because Moondream3
|
||||
pre-fills BOS and image embeddings together.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert "prompt_token_ids" in processed_inputs
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_pixel_values(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that pixel values are correctly produced."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
# Check mm_kwargs contains pixel_values
|
||||
mm_kwargs = processed_inputs.get("mm_kwargs")
|
||||
assert mm_kwargs is not None
|
||||
mm_data_result = mm_kwargs.get_data()
|
||||
assert "pixel_values" in mm_data_result
|
||||
|
||||
# Verify pixel_values shape
|
||||
pixel_values = mm_data_result["pixel_values"]
|
||||
assert pixel_values.dim() == 5 # [batch, num_crops, C, H, W]
|
||||
assert pixel_values.shape[2] == 3 # RGB channels
|
||||
assert pixel_values.shape[3] == 378 # crop height
|
||||
assert pixel_values.shape[4] == 378 # crop width
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_image_token_expansion(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that <image> placeholder is expanded to correct number of tokens."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe.<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_multi_crop_tiling(
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that large images produce correct multi-crop tiling."""
|
||||
from PIL import Image
|
||||
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
# Create a large image that requires multiple crops
|
||||
large_image = Image.new("RGB", (1000, 1000), color="blue")
|
||||
pixel_values, tiling = processor.preprocess_image(large_image)
|
||||
|
||||
# Large images should produce more than 1x1 tiling
|
||||
assert tiling[0] >= 1 and tiling[1] >= 1
|
||||
# Check that we have global crop + local crops
|
||||
expected_crops = tiling[0] * tiling[1] + 1
|
||||
assert pixel_values.shape[0] == expected_crops
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_size",
|
||||
[
|
||||
(500, 500),
|
||||
(800, 600),
|
||||
(1920, 1080),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_tiling_various_sizes(
|
||||
image_size: tuple[int, int],
|
||||
model_id: str,
|
||||
):
|
||||
"""Test tiling with various image sizes."""
|
||||
from PIL import Image
|
||||
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
width, height = image_size
|
||||
image = Image.new("RGB", (width, height), color="red")
|
||||
pixel_values, tiling = processor.preprocess_image(image)
|
||||
|
||||
# Basic shape checks
|
||||
assert pixel_values.dim() == 4 # [num_crops, C, H, W]
|
||||
assert pixel_values.shape[1] == 3 # RGB
|
||||
assert pixel_values.shape[2] == 378 # crop height
|
||||
assert pixel_values.shape[3] == 378 # crop width
|
||||
|
||||
# Tiling should respect max_crops (12)
|
||||
assert tiling[0] * tiling[1] <= 12
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_pixel_normalization(
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that pixel values are normalized to [-1, 1] range."""
|
||||
from PIL import Image
|
||||
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
# Create test image
|
||||
image = Image.new("RGB", (378, 378), color="green")
|
||||
pixel_values, _ = processor.preprocess_image(image)
|
||||
|
||||
# Normalization: (x - 0.5) / 0.5 = 2*x - 1
|
||||
# For input [0, 1], output should be [-1, 1]
|
||||
assert pixel_values.min() >= -1.0
|
||||
assert pixel_values.max() <= 1.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_chat_template_with_image(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that chat template correctly formats BOS + image + prompt."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
tokenizer = ctx.tokenizer
|
||||
|
||||
# Use the chat template format
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
token_ids = processed_inputs["prompt_token_ids"]
|
||||
|
||||
# BOS token (<|endoftext|>) should be token ID 0
|
||||
bos_token_id = tokenizer.encode("<|endoftext|>", add_special_tokens=False)[0]
|
||||
assert bos_token_id == 0
|
||||
|
||||
# First token should be BOS
|
||||
assert token_ids[0] == bos_token_id
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"content",
|
||||
[
|
||||
pytest.param(
|
||||
[
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.invalid/image.png"},
|
||||
},
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
],
|
||||
id="image-first",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.invalid/image.png"},
|
||||
},
|
||||
],
|
||||
id="text-first",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_chat_template_content_list_uses_moondream_image_prefix(
|
||||
image_assets: ImageTestAssets,
|
||||
content: list[dict[str, object]],
|
||||
model_id: str,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
|
||||
prompt = hf_processor.tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": content}],
|
||||
chat_template=hf_processor.chat_template,
|
||||
tokenize=False,
|
||||
)
|
||||
|
||||
expected_prompt = (
|
||||
"<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>"
|
||||
"What is in this image?<|md_reserved_2|>"
|
||||
)
|
||||
assert prompt == expected_prompt
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"image": [image_assets[0].pil_image]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_bos_token_always_first(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that BOS token (ID 0) is always at position 0."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Start with BOS token explicitly
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>Describe this image.<|md_reserved_2|>" # noqa: E501
|
||||
mm_data = {"image": [image_assets[0].pil_image]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
token_ids = processed_inputs["prompt_token_ids"]
|
||||
|
||||
# Token ID 0 (<|endoftext|>) should be the first token
|
||||
assert token_ids[0] == 0, (
|
||||
f"Expected BOS token (0) at position 0, got {token_ids[0]}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_with_small_image(
|
||||
model_id: str,
|
||||
):
|
||||
"""Test processor with image smaller than crop size."""
|
||||
from PIL import Image
|
||||
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
# Small image (smaller than crop size)
|
||||
small_image = Image.new("RGB", (100, 100), color="yellow")
|
||||
pixel_values, tiling = processor.preprocess_image(small_image)
|
||||
|
||||
# Small images should use 1x1 tiling
|
||||
assert tiling == (1, 1)
|
||||
# Should have 2 crops (global + 1 local)
|
||||
assert pixel_values.shape[0] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_kind",
|
||||
[
|
||||
pytest.param("numpy_hwc", id="numpy-hwc"),
|
||||
pytest.param("numpy_chw", id="numpy-chw"),
|
||||
pytest.param("torch_chw", id="torch-chw"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_preprocess_image_accepts_non_pil_inputs(
|
||||
image_assets: ImageTestAssets,
|
||||
image_kind: str,
|
||||
model_id: str,
|
||||
):
|
||||
from vllm.transformers_utils.processors.moondream3 import Moondream3Processor
|
||||
|
||||
processor = Moondream3Processor.from_pretrained(model_id, trust_remote_code=True)
|
||||
pil_image = image_assets[0].pil_image.convert("RGB")
|
||||
hwc_array = np.asarray(pil_image)
|
||||
expected_pixel_values, expected_tiling = processor.preprocess_image(pil_image)
|
||||
|
||||
if image_kind == "numpy_hwc":
|
||||
image = hwc_array
|
||||
elif image_kind == "numpy_chw":
|
||||
image = np.transpose(hwc_array, (2, 0, 1))
|
||||
else:
|
||||
image = torch.from_numpy(np.transpose(hwc_array, (2, 0, 1)).copy())
|
||||
|
||||
pixel_values, tiling = processor.preprocess_image(image)
|
||||
|
||||
assert tiling == expected_tiling
|
||||
assert pixel_values.shape == expected_pixel_values.shape
|
||||
assert pixel_values.dtype == torch.bfloat16
|
||||
assert torch.equal(pixel_values, expected_pixel_values)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("image_kind", ["numpy_chw", "torch_chw"])
|
||||
@pytest.mark.parametrize("model_id", [MOONDREAM3_MODEL_ID])
|
||||
def test_processor_apply_accepts_non_pil_image_inputs(
|
||||
image_assets: ImageTestAssets,
|
||||
image_kind: str,
|
||||
model_id: str,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What is this?<|md_reserved_2|>" # noqa: E501
|
||||
hwc_array = np.asarray(image_assets[0].pil_image.convert("RGB"))
|
||||
chw_array = np.transpose(hwc_array, (2, 0, 1)).copy()
|
||||
image = chw_array if image_kind == "numpy_chw" else torch.from_numpy(chw_array)
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"image": [image]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
image_placeholders = processed_inputs["mm_placeholders"]["image"]
|
||||
assert len(image_placeholders) == 1
|
||||
assert image_placeholders[0].length == EXPECTED_IMAGE_TOKENS
|
||||
|
||||
mm_kwargs = processed_inputs["mm_kwargs"].get_data()
|
||||
assert mm_kwargs["pixel_values"].shape[2:] == (3, 378, 378)
|
||||
|
||||
|
||||
class TestMoondream3TilingLogic:
|
||||
"""CPU-based tests for Moondream3 tiling selection logic.
|
||||
|
||||
These tests validate the select_tiling() function which determines
|
||||
how images are divided into crops for the vision encoder.
|
||||
"""
|
||||
|
||||
def test_small_image_no_tiling(self):
|
||||
"""Small images should use 1x1 tiling."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=300, width=300, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
assert tiling == (1, 1)
|
||||
|
||||
def test_exact_crop_size(self):
|
||||
"""Image exactly at crop size should use 1x1."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=CROP_SIZE, width=CROP_SIZE, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
assert tiling == (1, 1)
|
||||
|
||||
def test_large_square_image(self):
|
||||
"""Large square image should use multiple tiles."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=800, width=800, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
h_tiles, w_tiles = tiling
|
||||
assert h_tiles >= 2
|
||||
assert w_tiles >= 2
|
||||
assert h_tiles * w_tiles <= MAX_CROPS
|
||||
|
||||
def test_wide_image(self):
|
||||
"""Wide image should have more width tiles."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=400, width=1200, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
h_tiles, w_tiles = tiling
|
||||
assert w_tiles >= h_tiles
|
||||
|
||||
def test_tall_image(self):
|
||||
"""Tall image should have more height tiles."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=1200, width=400, crop_size=CROP_SIZE, max_crops=MAX_CROPS
|
||||
)
|
||||
h_tiles, w_tiles = tiling
|
||||
assert h_tiles >= w_tiles
|
||||
|
||||
def test_respects_max_crops(self):
|
||||
"""Tiling should not exceed max_crops."""
|
||||
from vllm.transformers_utils.processors.moondream3 import select_tiling
|
||||
|
||||
tiling = select_tiling(
|
||||
height=2000, width=2000, crop_size=CROP_SIZE, max_crops=4
|
||||
)
|
||||
h_tiles, w_tiles = tiling
|
||||
assert h_tiles * w_tiles <= 4
|
||||
|
||||
|
||||
class TestMoondream3VisionShapes:
|
||||
"""CPU-based tests for vision encoder expected shapes.
|
||||
|
||||
These tests verify the mathematical relationships between
|
||||
crop size, patch size, and token counts.
|
||||
"""
|
||||
|
||||
def test_expected_patch_count(self):
|
||||
"""Test 378/14 = 27 patches per side, 729 total."""
|
||||
patches_per_side = CROP_SIZE // PATCH_SIZE
|
||||
total_patches = patches_per_side**2
|
||||
|
||||
assert patches_per_side == 27
|
||||
assert total_patches == EXPECTED_IMAGE_TOKENS - 1
|
||||
|
||||
def test_patch_embedding_input_dim(self):
|
||||
"""Test patch embedding input dimension."""
|
||||
channels = 3
|
||||
input_dim = PATCH_SIZE * PATCH_SIZE * channels
|
||||
|
||||
assert input_dim == 14 * 14 * 3
|
||||
assert input_dim == 588
|
||||
|
||||
|
||||
class TestMoondream3TauAttention:
|
||||
"""CPU-based tests for tau attention scaling components.
|
||||
|
||||
These tests validate the tau attention formula used in Moondream3:
|
||||
- Token-based: tok_q = tanh(gelu(qkv) @ tau_wq.T)
|
||||
- Position-based: tau_pos = 1 + (sigmoid(alpha * log(pos+1)) - 0.5)
|
||||
"""
|
||||
|
||||
def test_tau_position_range(self):
|
||||
"""Test tau position scaling produces values in valid range."""
|
||||
num_heads = 32
|
||||
seq_len = 100
|
||||
|
||||
tau_alpha = torch.randn(num_heads)
|
||||
positions = torch.arange(seq_len)
|
||||
|
||||
pos_float = (positions.float() + 1.0).clamp(min=1e-6)
|
||||
pos_log = pos_float.log()
|
||||
tau_pos = 1.0 + (torch.sigmoid(tau_alpha[:, None] * pos_log[None, :]) - 0.5)
|
||||
|
||||
assert tau_pos.shape == (num_heads, seq_len)
|
||||
# tau_pos should be between 0.5 and 1.5
|
||||
assert tau_pos.min() >= 0.5
|
||||
assert tau_pos.max() <= 1.5
|
||||
|
||||
def test_tau_token_output_range(self):
|
||||
"""Test tau token scaling output is bounded by tanh."""
|
||||
import torch.nn.functional as F
|
||||
|
||||
seq_len = 100
|
||||
qkv_dim = 6144 # 2048 * 3
|
||||
num_heads = 32
|
||||
|
||||
qkv = torch.randn(seq_len, qkv_dim)
|
||||
tau_wq = torch.randn(num_heads, qkv_dim)
|
||||
|
||||
tok_feat = F.gelu(qkv)
|
||||
tok_q = torch.tanh(tok_feat @ tau_wq.t())
|
||||
|
||||
assert tok_q.shape == (seq_len, num_heads)
|
||||
# tanh output is bounded by [-1, 1]
|
||||
assert tok_q.min() >= -1.0
|
||||
assert tok_q.max() <= 1.0
|
||||
@@ -1,137 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Nemotron-Nano-VL's multimodal preprocessing kwargs."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.image import rescale_image_size
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
def _get_expected_num_patches(
|
||||
config: PreTrainedConfig,
|
||||
image: Image.Image,
|
||||
num_imgs: int,
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
):
|
||||
from vllm.transformers_utils.processors.nemotron_vl import (
|
||||
calculate_nemotron_vl_targets,
|
||||
get_nemotron_vl_target_ratios,
|
||||
)
|
||||
|
||||
width, height = image.size
|
||||
|
||||
blocks, _, _ = calculate_nemotron_vl_targets(
|
||||
orig_width=width,
|
||||
orig_height=height,
|
||||
target_ratios=get_nemotron_vl_target_ratios(
|
||||
min_num,
|
||||
max_num,
|
||||
),
|
||||
image_size=config.force_image_size,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
expected_num_patches = blocks
|
||||
|
||||
if config.use_thumbnail and expected_num_patches > 1:
|
||||
expected_num_patches += 1
|
||||
|
||||
return expected_num_patches
|
||||
|
||||
|
||||
def _run_check(
|
||||
processor: BaseMultiModalProcessor,
|
||||
images: list[Image.Image],
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
mm_processor_kwargs: Mapping[str, object],
|
||||
):
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
config = processor.info.get_hf_config()
|
||||
image_processor = processor.info.get_image_processor()
|
||||
|
||||
config.use_thumbnail = image_processor.use_thumbnail
|
||||
prompt = "<image>" * len(images)
|
||||
mm_data = {"image": images}
|
||||
|
||||
total_expected_num_patches = sum(
|
||||
_get_expected_num_patches(config, image, len(images), min_num, max_num)
|
||||
for image in images
|
||||
)
|
||||
print(total_expected_num_patches)
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
image_token_id = tokenizer.convert_tokens_to_ids("<image>")
|
||||
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
|
||||
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values_flat"].shape
|
||||
print("Image token count:", img_tok_count, "Pixel shape:", pixel_shape)
|
||||
assert img_tok_count == 256 * total_expected_num_patches
|
||||
assert pixel_shape[0] == total_expected_num_patches
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1"])
|
||||
@pytest.mark.parametrize(
|
||||
"size_factors",
|
||||
[
|
||||
# Single-scale
|
||||
[1.0],
|
||||
# Single-scale, batched
|
||||
[1.0, 1.0, 1.0],
|
||||
# Multi-scale
|
||||
[0.25, 0.5, 1.0],
|
||||
[4.0, 2.0, 1.0],
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("min_dynamic_patch", "max_dynamic_patch"),
|
||||
[(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
|
||||
)
|
||||
@pytest.mark.parametrize("dynamic_image_size", [True, False])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
model_id: str,
|
||||
image_assets: ImageTestAssets,
|
||||
size_factors: list[int],
|
||||
min_dynamic_patch: int,
|
||||
max_dynamic_patch: int,
|
||||
dynamic_image_size: bool | None,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
mm_processor_kwargs = {
|
||||
"min_dynamic_patch": min_dynamic_patch,
|
||||
"max_dynamic_patch": max_dynamic_patch,
|
||||
"dynamic_image_size": dynamic_image_size,
|
||||
}
|
||||
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": len(size_factors)},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
|
||||
min_num = min_dynamic_patch if dynamic_image_size else 1
|
||||
max_num = max_dynamic_patch if dynamic_image_size else 1
|
||||
|
||||
_run_check(
|
||||
processor,
|
||||
[rescale_image_size(image_assets[0].pil_image, f) for f in size_factors],
|
||||
min_num,
|
||||
max_num,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
@@ -1,181 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Qwen3-ASR preprocessing must preserve its HF processor's audio inputs."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.models.qwen3_asr import Qwen3ASRProcessor
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import batched_tensors_equal
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
|
||||
from .test_common import _assert_inputs_equal
|
||||
|
||||
_AUDIO_PROMPT = "<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
|
||||
|
||||
@pytest.fixture(params=["Qwen/Qwen3-ASR-0.6B", "Qwen/Qwen3-ASR-0.6B-hf"])
|
||||
def asr_processor(request):
|
||||
config = ModelConfig(
|
||||
request.param,
|
||||
dtype="float32",
|
||||
max_model_len=4096,
|
||||
limit_mm_per_prompt={"audio": 2},
|
||||
mm_processor_cache_gb=0,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(config)
|
||||
native = Qwen3ASRProcessor.from_pretrained(request.param)
|
||||
return processor, native
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"samples", [321, 15999, 16000, 16001, 16159, 16160, 16161, 32001, 480001]
|
||||
)
|
||||
@pytest.mark.parametrize("num_audios", [1, 2])
|
||||
def test_audio_processing_matches_hf(asr_processor, samples, num_audios):
|
||||
"""Each clip must match HF independently of other clips in the request."""
|
||||
processor, native = asr_processor
|
||||
rng = np.random.RandomState(42)
|
||||
audios = [
|
||||
rng.randn(samples + index * 160).astype(np.float32)
|
||||
for index in range(num_audios)
|
||||
]
|
||||
expected = [
|
||||
native(text=_AUDIO_PROMPT, audio=audio, return_tensors="pt") for audio in audios
|
||||
]
|
||||
actual = processor(
|
||||
_AUDIO_PROMPT * num_audios,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
expected_ids = [
|
||||
token for item in expected for token in item["input_ids"][0].tolist()
|
||||
]
|
||||
assert actual["prompt_token_ids"] == expected_ids
|
||||
masks = [item["input_features_mask"][0] for item in expected]
|
||||
expected_features = torch.cat(
|
||||
[
|
||||
item["input_features"][0, :, mask.bool()]
|
||||
for item, mask in zip(expected, masks)
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
actual_features = actual["mm_kwargs"].get_data()
|
||||
torch.testing.assert_close(
|
||||
actual_features["input_audio_features"], expected_features, rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual_features["audio_feature_lengths"],
|
||||
torch.stack([mask.sum() for mask in masks]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cached_indices", [[], [0], [1], [0, 1]])
|
||||
def test_audio_processing_cache_is_batch_independent(asr_processor, cached_indices):
|
||||
"""Partial cache hits must not change features, masks, or prompt expansion."""
|
||||
processor, _ = asr_processor
|
||||
config = processor.info.ctx.model_config
|
||||
config.get_multimodal_config().mm_processor_cache_gb = 1
|
||||
cache = MultiModalProcessorOnlyCache(config)
|
||||
rng = np.random.RandomState(42)
|
||||
audios = [rng.randn(samples).astype(np.float32) for samples in (16001, 32000)]
|
||||
for index in cached_indices:
|
||||
processor(
|
||||
_AUDIO_PROMPT,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audios[index]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
for batch in (audios, audios[::-1]):
|
||||
kwargs = dict(
|
||||
mm_items=processor.info.parse_mm_data({"audio": batch}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
_assert_inputs_equal(
|
||||
processor(_AUDIO_PROMPT * 2, **kwargs),
|
||||
processor(_AUDIO_PROMPT * 2, **kwargs, cache=cache),
|
||||
)
|
||||
|
||||
|
||||
def test_text_only_processing(asr_processor):
|
||||
"""Empty media input must not invoke the audio feature extractor."""
|
||||
processor, native = asr_processor
|
||||
prompt = "Transcribe the audio."
|
||||
actual = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert actual["prompt_token_ids"] == native.tokenizer.encode(prompt)
|
||||
assert actual["mm_kwargs"].get_data() == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hf_data", [{}, {"text": ""}, {"audio": None}, {"audio": []}])
|
||||
def test_hf_processor_without_audio(asr_processor, hf_data):
|
||||
"""The media hook must not require audio or call HF for absent waveforms."""
|
||||
processor, _ = asr_processor
|
||||
assert not processor._call_hf_processor(hf_data, {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hf_data", [{}, {"text": ""}])
|
||||
def test_postprocessing_without_audio_features(asr_processor, hf_data):
|
||||
"""Non-audio output must pass through even when HF input is nonempty."""
|
||||
processor, _ = asr_processor
|
||||
processed = BatchFeature({"input_ids": torch.tensor([[1, 2]])})
|
||||
actual = processor._postprocess_hf_mm_data(hf_data, {}, processed)
|
||||
assert actual is processed
|
||||
assert set(actual) == {"input_ids"}
|
||||
torch.testing.assert_close(actual["input_ids"], torch.tensor([[1, 2]]))
|
||||
|
||||
|
||||
def test_chat_template_processing(asr_processor):
|
||||
"""The generic processor loader must still render audio chat prompts."""
|
||||
processor, native = asr_processor
|
||||
chat_processor = cached_processor_from_config(processor.info.ctx.model_config)
|
||||
messages = [
|
||||
{"role": "system", "content": ""},
|
||||
{"role": "user", "content": [{"type": "audio", "audio": "unused"}]},
|
||||
]
|
||||
prompt = chat_processor.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
expected_prompt = native.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
assert prompt == expected_prompt
|
||||
audio = np.random.RandomState(42).randn(16001).astype(np.float32)
|
||||
expected = native(text=expected_prompt, audio=audio, return_tensors="pt")
|
||||
actual = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audio}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert actual["prompt_token_ids"] == expected["input_ids"][0].tolist()
|
||||
|
||||
|
||||
def test_precomputed_audio_features(asr_processor):
|
||||
"""Precomputed features must bypass waveform extraction unchanged."""
|
||||
processor, _ = asr_processor
|
||||
processor.info.ctx.get_mm_config().enable_mm_embeds = True
|
||||
audio = np.random.RandomState(42).randn(16001).astype(np.float32)
|
||||
expected = processor(
|
||||
_AUDIO_PROMPT,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audio}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
features = expected["mm_kwargs"].get_data()
|
||||
actual = processor(
|
||||
_AUDIO_PROMPT,
|
||||
mm_items=processor.info.parse_mm_data({"audio": features}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert actual["prompt_token_ids"] == expected["prompt_token_ids"]
|
||||
assert actual["mm_placeholders"] == expected["mm_placeholders"]
|
||||
assert batched_tensors_equal(actual["mm_kwargs"].get_data(), features)
|
||||
@@ -1,140 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Qwen3 Omni audio processing and sample rate handling."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Qwen/Qwen3-Omni-30B-A3B-Instruct"])
|
||||
@pytest.mark.parametrize(
|
||||
("audio_sample_rate", "audio_duration_sec"),
|
||||
[
|
||||
(16000, 1.0), # Native Whisper sample rate, 1 second
|
||||
(16000, 2.0), # Native Whisper sample rate, 2 seconds
|
||||
],
|
||||
)
|
||||
def test_processor_with_audio_sample_rate(
|
||||
model_id: str,
|
||||
audio_sample_rate: int,
|
||||
audio_duration_sec: float,
|
||||
) -> None:
|
||||
"""Test that vLLM's processor generates expected outputs with audio_sample_rate.
|
||||
|
||||
This validates that the processor correctly handles audio_sample_rate
|
||||
passed via hf_processor_mm_kwargs and generates audio tokens.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 0},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
|
||||
# Create audio data at the specified sample rate
|
||||
audio_length = int(audio_sample_rate * audio_duration_sec)
|
||||
rng = np.random.RandomState(42)
|
||||
audio_data = rng.rand(audio_length).astype(np.float32)
|
||||
|
||||
# Build prompt with audio placeholder
|
||||
prompt = "<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
mm_data = {"audio": [(audio_data, audio_sample_rate)]}
|
||||
|
||||
# Apply processor with audio_sample_rate in mm_kwargs
|
||||
hf_processor_mm_kwargs: dict[str, Any] = {
|
||||
"audio_sample_rate": audio_sample_rate,
|
||||
}
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# Verify audio tokens are generated
|
||||
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
audio_token_id = tokenizer.convert_tokens_to_ids(hf_processor.audio_token)
|
||||
aud_tok_count = processed_inputs["prompt_token_ids"].count(audio_token_id)
|
||||
|
||||
assert aud_tok_count >= 1, (
|
||||
f"Expected at least 1 audio token but got {aud_tok_count}. "
|
||||
f"sample_rate: {audio_sample_rate}Hz, duration: {audio_duration_sec}s"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Qwen/Qwen3-Omni-30B-A3B-Instruct"])
|
||||
def test_longer_audio_generates_more_tokens(model_id: str) -> None:
|
||||
"""Test that longer audio generates more tokens than shorter audio.
|
||||
|
||||
This validates that audio_sample_rate is being used correctly by checking
|
||||
that audio duration affects token count as expected.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 0},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
|
||||
audio_sample_rate = 16000
|
||||
rng = np.random.RandomState(42)
|
||||
|
||||
def get_token_count(duration: float) -> int:
|
||||
audio_length = int(audio_sample_rate * duration)
|
||||
audio_data = rng.rand(audio_length).astype(np.float32)
|
||||
prompt = "<|audio_start|><|audio_pad|><|audio_end|>"
|
||||
mm_data = {"audio": [(audio_data, audio_sample_rate)]}
|
||||
hf_processor_mm_kwargs: dict[str, Any] = {
|
||||
"audio_sample_rate": audio_sample_rate,
|
||||
}
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
hf_proc = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
audio_token_id = tokenizer.convert_tokens_to_ids(hf_proc.audio_token)
|
||||
return processed["prompt_token_ids"].count(audio_token_id)
|
||||
|
||||
short_tokens = get_token_count(1.0)
|
||||
long_tokens = get_token_count(2.0)
|
||||
|
||||
assert long_tokens > short_tokens, (
|
||||
f"Expected longer audio (2s) to have more tokens than shorter (1s). "
|
||||
f"Got short={short_tokens}, long={long_tokens}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Cloudflare/clef-omni"])
|
||||
def test_audio_in_video_follows_requested_fps(model_id: str) -> None:
|
||||
"""Audio and video tokens interleave by the fps the video was processed
|
||||
at, as in HF, not by the default of one frame a second."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
rng = np.random.RandomState(42)
|
||||
video = rng.randint(0, 255, (4, 64, 64, 3), dtype=np.uint8)
|
||||
audio = rng.rand(32000).astype(np.float32)
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
mm_kwargs: dict[str, Any] = {
|
||||
"use_audio_in_video": True,
|
||||
"videos_kwargs": {"fps": 2.0, "do_sample_frames": False},
|
||||
}
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(
|
||||
{"video": [video], "audio": [(audio, 16000)]}
|
||||
),
|
||||
hf_processor_mm_kwargs=mm_kwargs,
|
||||
)
|
||||
expected = processor.info.get_hf_processor()(
|
||||
text=[prompt], videos=[video], audio=[audio], return_tensors="pt", **mm_kwargs
|
||||
)
|
||||
assert processed["prompt_token_ids"] == expected["input_ids"][0].tolist()
|
||||
@@ -1,85 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for smolvlm's multimodal preprocessing kwargs."""
|
||||
|
||||
import pytest
|
||||
from transformers import SmolVLMConfig
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["HuggingFaceTB/SmolVLM2-2.2B-Instruct"])
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_toks_per_img"),
|
||||
[
|
||||
({"max_image_size": {"longest_edge": 384}}, 1377),
|
||||
({"max_image_size": {"longest_edge": 768}}, 405),
|
||||
({"do_image_splitting": False}, 81),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
expected_toks_per_img: int,
|
||||
num_imgs: int,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
"""Ensure Idefics3MultiModalProcessor handles num_crops properly."""
|
||||
# Same as the previous test - don't initialize mm_processor_kwargs
|
||||
# in this test and assume that the kwargs will be correctly expanded by
|
||||
# the partial when calling the custom input processor.
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
|
||||
# Build the image str / prompt based on the number of images we pass
|
||||
placeholders = (
|
||||
"<image>"
|
||||
if num_imgs == 1
|
||||
else "\n".join(f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
|
||||
)
|
||||
prompt = f"<|im_start|>User:{placeholders}\n<end_of_utterance>\nAssistant:" # noqa: E501
|
||||
|
||||
# Build mm_data
|
||||
image_size = ctx.get_hf_config(SmolVLMConfig).vision_config.image_size
|
||||
dummy_image_size = (image_size * 4, image_size * 4)
|
||||
dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
|
||||
mm_data = {"image": [dummy_image] * num_imgs}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# Ensure the placeholders format are correct
|
||||
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
num_patches = processor.info.get_num_patches(
|
||||
image_width=dummy_image.width,
|
||||
image_height=dummy_image.height,
|
||||
processor=hf_processor,
|
||||
mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
assert num_patches == expected_toks_per_img // hf_processor.image_seq_len
|
||||
|
||||
hf_processed_inputs = hf_processor(
|
||||
text=prompt,
|
||||
images=mm_data["image"],
|
||||
**processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs),
|
||||
)
|
||||
assert processed_inputs["prompt_token_ids"] == hf_processed_inputs["input_ids"][0]
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
image_token_id = ctx.get_hf_config().image_token_id
|
||||
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
|
||||
assert img_tok_count == expected_toks_per_img * num_imgs
|
||||
@@ -1,37 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Whisper's multimodal preprocessing."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["openai/whisper-large-v3-turbo"])
|
||||
@pytest.mark.parametrize("audio_duration_s", [5, 30, 35])
|
||||
def test_audio_features_fit_encoder_window(
|
||||
model_id: str,
|
||||
audio_duration_s: int,
|
||||
) -> None:
|
||||
"""The encoder only has positions for one 30 s window, so a longer clip
|
||||
has to be cut to that window like the HF feature extractor does by default.
|
||||
"""
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": 1})
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
feature_extractor = processor.info.get_feature_extractor()
|
||||
|
||||
sampling_rate = feature_extractor.sampling_rate
|
||||
rng = np.random.RandomState(0)
|
||||
audio = rng.rand(sampling_rate * audio_duration_s).astype(np.float32)
|
||||
|
||||
processed_inputs = processor(
|
||||
"<|startoftranscript|>",
|
||||
mm_items=processor.info.parse_mm_data({"audio": [(audio, sampling_rate)]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
input_features = processed_inputs["mm_kwargs"].get_data()["input_features"]
|
||||
assert input_features.shape[-1] == feature_extractor.nb_max_frames
|
||||
@@ -1,164 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal.image import rescale_image_size
|
||||
|
||||
from ...conftest import IMAGE_ASSETS, ImageTestAssets, VllmRunner
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<|im_start|>User\n<image>\nWhat's the content in the center of the image?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
"cherry_blossom": "<|im_start|>User\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
}
|
||||
)
|
||||
|
||||
IMAGE_SIZE_FACTOR_GROUPS = (
|
||||
# Single-scale
|
||||
(1.0,),
|
||||
# Single-scale, batched
|
||||
(1.0, 1.0, 1.0),
|
||||
# Multi-scale
|
||||
(0.25, 0.5, 1.0),
|
||||
)
|
||||
|
||||
|
||||
def run_awq_test(
|
||||
vllm_runner: type[VllmRunner],
|
||||
image_assets: ImageTestAssets,
|
||||
source_model: str,
|
||||
quant_model: str,
|
||||
*,
|
||||
size_factor_groups: tuple[tuple[float, ...], ...],
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
tensor_parallel_size: int,
|
||||
distributed_executor_backend: str | None = None,
|
||||
):
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
|
||||
inputs_per_image_and_size_group = [
|
||||
(
|
||||
[prompt for _ in size_factors],
|
||||
[rescale_image_size(image, factor) for factor in size_factors],
|
||||
)
|
||||
for size_factors in size_factor_groups
|
||||
for image, prompt in zip(images, HF_IMAGE_PROMPTS)
|
||||
]
|
||||
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
|
||||
# max_model_len should be greater than image_feature_size
|
||||
with vllm_runner(
|
||||
source_model,
|
||||
max_model_len=4096,
|
||||
dtype=dtype,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enforce_eager=True,
|
||||
default_torch_num_threads=1,
|
||||
) as vllm_model:
|
||||
source_outputs_per_image = [
|
||||
vllm_model.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs=num_logprobs, images=images
|
||||
)
|
||||
for prompts, images in inputs_per_image_and_size_group
|
||||
]
|
||||
|
||||
with vllm_runner(
|
||||
quant_model,
|
||||
quantization="awq",
|
||||
max_model_len=4096,
|
||||
dtype=dtype,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enforce_eager=True,
|
||||
default_torch_num_threads=1,
|
||||
) as vllm_model:
|
||||
quant_outputs_per_image = [
|
||||
vllm_model.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs=num_logprobs, images=images
|
||||
)
|
||||
for prompts, images in inputs_per_image_and_size_group
|
||||
]
|
||||
|
||||
for source_outputs, quant_outputs in zip(
|
||||
source_outputs_per_image, quant_outputs_per_image
|
||||
):
|
||||
# TODO: Check whether using original CLIPVisionModel can improve
|
||||
# consistency against HF
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=source_outputs,
|
||||
outputs_1_lst=quant_outputs,
|
||||
name_0="source",
|
||||
name_1="awq",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "quantization", "dtype"),
|
||||
[
|
||||
("mattbucci/gemma-4-26B-AWQ", "awq", "float16"),
|
||||
("cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit", "compressed-tensors", "bfloat16"),
|
||||
],
|
||||
ids=[
|
||||
"gemma4-moe-standard-awq-dot-suffix",
|
||||
"gemma4-moe-compressed-tensors-underscore-suffix",
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_awq_load(
|
||||
vllm_runner: type[VllmRunner],
|
||||
example_prompts: list[str],
|
||||
model: str,
|
||||
quantization: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Regression test: AWQ weight loading must not KeyError."""
|
||||
with vllm_runner(
|
||||
model,
|
||||
quantization=quantization,
|
||||
dtype=dtype,
|
||||
max_model_len=128,
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
outputs = vllm_model.generate_greedy(example_prompts[:2], max_tokens=32)
|
||||
assert len(outputs) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source_model", "quant_model"),
|
||||
[("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@torch.inference_mode()
|
||||
def test_awq_models(
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
source_model,
|
||||
quant_model,
|
||||
dtype,
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
) -> None:
|
||||
run_awq_test(
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
source_model,
|
||||
quant_model,
|
||||
size_factor_groups=IMAGE_SIZE_FACTOR_GROUPS,
|
||||
dtype=dtype,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for online FP8 per-channel quantization.
|
||||
|
||||
Loads a BF16 model with ``--quantization fp8_per_channel`` (online
|
||||
quantization) and compares log-probabilities against the same model served in
|
||||
BF16 without quantization. This exercises the full pipeline: config parsing,
|
||||
``Fp8PtpcOnlineLinearMethod``, ``Fp8PtpcOnlineMoEMethod``, weight
|
||||
loading, online quantization / shuffling, and inference.
|
||||
|
||||
``example_prompts`` is a pytest fixture (from conftest.py) that loads 8
|
||||
diverse prompts from ``tests/prompts/example.txt``.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
# Small MoE model that fits on a single GPU and exercises both linear + MoE.
|
||||
MOE_MODEL = "allenai/OLMoE-1B-7B-0125-Instruct"
|
||||
# Small dense model (no MoE) to validate the linear-only path.
|
||||
DENSE_MODEL = "Qwen/Qwen3-0.6B"
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
MAX_TOKENS = 4
|
||||
NUM_LOG_PROBS = 8
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="fp8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
|
||||
def test_fp8_per_channel_logprobs(
|
||||
vllm_runner,
|
||||
quant_baseline_logprobs,
|
||||
example_prompts,
|
||||
model: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Compare BF16 baseline logprobs against online per-channel-quantized
|
||||
model.
|
||||
|
||||
Runs the same model twice -- once in BF16 (baseline) and once with online
|
||||
FP8 per-channel quantization -- then checks that the top log-probabilities
|
||||
are close. Only 4 tokens are generated to keep the test fast while still
|
||||
catching numerical divergence beyond expected per-channel error.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
|
||||
baseline_outputs = quant_baseline_logprobs(
|
||||
model,
|
||||
example_prompts,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
max_tokens=MAX_TOKENS,
|
||||
num_logprobs=NUM_LOG_PROBS,
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
enforce_eager=True,
|
||||
quantization="fp8_per_channel",
|
||||
) as vllm_model:
|
||||
test_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline_outputs,
|
||||
outputs_1_lst=test_outputs,
|
||||
name_0="bf16",
|
||||
name_1="fp8_per_channel",
|
||||
)
|
||||
@@ -1,176 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Routing support for the AITER Triton MXFP4 W4A16 MoE backend.
|
||||
|
||||
MiMo-V2.6 uses an ungrouped sigmoid router with a per-expert correction bias
|
||||
(``scoring_func=sigmoid``, ``topk_method=noaux_tc``, ``n_group == 1``), which
|
||||
``get_routing_method_type`` classifies as ``RoutingMethodType.DeepSeekV3``.
|
||||
``AiterW4A16ExpertsMonolithic`` used to reject that router ("kernel does not
|
||||
support routing method ..."), and on gfx942 it is the only MXFP4 kernel whose
|
||||
device gate accepts the card, so the checkpoint could not be served natively at
|
||||
all.
|
||||
|
||||
aiter's *flat* top-k handles that router natively (``score_mode="sigmoid"`` plus
|
||||
a correction bias), so the backend only has to ask for it. This module covers
|
||||
both halves: the gate accepts the router family, and the routing the wrapper
|
||||
requests reproduces the model's own math.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
get_routing_method_type,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
|
||||
|
||||
MIMO_EXPERTS = 256 # MiMo-V2.6-Flash
|
||||
MIMO_PRO_EXPERTS = 384 # MiMo-V2.6-Pro
|
||||
MIMO_TOP_K = 8
|
||||
|
||||
|
||||
def mimo_routing_method() -> RoutingMethodType:
|
||||
"""MiMo-V2.6: sigmoid + correction bias, a single expert group."""
|
||||
return get_routing_method_type(
|
||||
scoring_func="sigmoid",
|
||||
top_k=MIMO_TOP_K,
|
||||
renormalize=True,
|
||||
num_expert_group=1,
|
||||
has_e_score_bias=True,
|
||||
)
|
||||
|
||||
|
||||
def mimo_moe_config(num_experts: int = MIMO_EXPERTS) -> FusedMoEConfig:
|
||||
return FusedMoEConfig(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=MIMO_TOP_K,
|
||||
hidden_dim=4096,
|
||||
intermediate_size=2048,
|
||||
num_local_experts=num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device=torch.device("cuda"),
|
||||
routing_method=mimo_routing_method(),
|
||||
router_logits_dtype=torch.float32,
|
||||
)
|
||||
|
||||
|
||||
def test_ungrouped_sigmoid_router_is_deepseekv3() -> None:
|
||||
"""Guard the premise of the fix: MiMo's router is DeepSeekV3-classified."""
|
||||
assert mimo_routing_method() == RoutingMethodType.DeepSeekV3
|
||||
|
||||
|
||||
def test_aiter_w4a16_supports_mimo_routing() -> None:
|
||||
"""The gate accepts MiMo's router family — the reported construction abort.
|
||||
|
||||
Only the predicate the fix touches is exercised, so this runs on any host;
|
||||
the full config path is covered by the test below.
|
||||
"""
|
||||
assert AiterW4A16ExpertsMonolithic._supports_routing_method(
|
||||
RoutingMethodType.DeepSeekV3, kMxfp4Static, None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="only runs on ROCm with a supported AITER install",
|
||||
)
|
||||
def test_aiter_w4a16_supports_mimo_config(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The same gate through the real config path, device check included."""
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
supported, reason = AiterW4A16ExpertsMonolithic.is_supported_config(
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
mimo_moe_config(),
|
||||
kMxfp4Static,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
|
||||
assert supported, reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_experts", [MIMO_EXPERTS, MIMO_PRO_EXPERTS])
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="only runs on ROCm with a supported AITER install",
|
||||
)
|
||||
def test_flat_sigmoid_routing_matches_mimo_reference(num_experts: int) -> None:
|
||||
"""The routing the wrapper requests must equal MiMo's own router math.
|
||||
|
||||
MiMo's router is ungrouped: sigmoid scores, selection on
|
||||
``sigmoid(logits) + bias``, weights = the *unbiased* sigmoid scores, then
|
||||
renormalized. Both checkpoints use it — 256 experts is Flash, 384 is Pro.
|
||||
|
||||
The ``sigmoid`` score mode comes from ROCm/aiter#4688 (aiter >= 0.1.20).
|
||||
"""
|
||||
aiter_routing = pytest.importorskip(
|
||||
"aiter.ops.triton.moe.moe_routing.routing"
|
||||
).routing
|
||||
|
||||
torch.manual_seed(0)
|
||||
top_k, num_tokens = MIMO_TOP_K, 16
|
||||
logits = torch.randn(num_tokens, num_experts, device="cuda") * 2.0
|
||||
bias = torch.randn(num_experts, device="cuda") * 0.5
|
||||
|
||||
# Reference: select on the biased score, return the unbiased score, renorm.
|
||||
scores = torch.sigmoid(logits.float())
|
||||
ref_ids = (scores + bias).topk(top_k, dim=-1).indices
|
||||
ref_weights = scores.gather(1, ref_ids)
|
||||
ref_weights = ref_weights / ref_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
def routed(score_mode: str) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
routing_data, topk_indx, _ = aiter_routing(
|
||||
logits,
|
||||
top_k,
|
||||
score_mode=score_mode,
|
||||
bias=bias,
|
||||
renorm=True,
|
||||
routed_scaling_factor=1.0,
|
||||
use_grouped_topk=False,
|
||||
)
|
||||
# `topk_indx[i]` is the token-slot index (token * top_k + slot) of the
|
||||
# i-th entry of the expert-sorted array whose weights are gate_scal.
|
||||
hist = routing_data.expt_hist.to(torch.long).cpu()
|
||||
expert_of_slot = torch.repeat_interleave(torch.arange(hist.numel()), hist)
|
||||
slot = topk_indx.reshape(-1).to(torch.long).cpu()
|
||||
weights = routing_data.gate_scal.reshape(-1).to(torch.float32).cpu()
|
||||
ids = torch.full((num_tokens, top_k), -1, dtype=torch.long)
|
||||
out = torch.zeros(num_tokens, top_k)
|
||||
for i in range(slot.numel()):
|
||||
token, position = int(slot[i]) // top_k, int(slot[i]) % top_k
|
||||
ids[token, position] = int(expert_of_slot[i])
|
||||
out[token, position] = float(weights[i])
|
||||
return ids, out
|
||||
|
||||
got_ids, got_weights = routed("sigmoid")
|
||||
|
||||
# aiter's triton top-k emits each token's selection sorted by *ascending
|
||||
# expert id*, while the reference is in descending ``sigmoid(logits)+bias``
|
||||
# rank order — align both sides on the expert id so the weight check
|
||||
# becomes elementwise and still validates the expert<->weight pairing.
|
||||
def align(ids: torch.Tensor, w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
order = torch.argsort(ids, dim=-1)
|
||||
return ids.gather(1, order), w.gather(1, order)
|
||||
|
||||
got_ids, got_weights = align(got_ids, got_weights)
|
||||
ref_ids, ref_weights = align(ref_ids.cpu(), ref_weights.cpu())
|
||||
assert torch.equal(got_ids, ref_ids)
|
||||
assert torch.allclose(got_weights, ref_weights, atol=1e-3)
|
||||
|
||||
# Control: the correction bias must steer selection only. Ignoring it would
|
||||
# pick from the plain sigmoid scores instead.
|
||||
plain_ids = scores.topk(top_k, dim=-1).indices
|
||||
assert not torch.equal(torch.sort(plain_ids.cpu(), dim=-1).values, ref_ids)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,93 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU-only tests for AliceAI configuration."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.models.config import AliceAIForCausalLMConfig
|
||||
|
||||
|
||||
def _make_config(**overrides) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
**{
|
||||
"kda_allow_negative_eigenvalues": False,
|
||||
"router_score_function": "sigmoid",
|
||||
"router_bias_correction": True,
|
||||
**overrides,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_model_config_warns_when_mtp_is_ignored(caplog) -> None:
|
||||
config = _make_config(mtp_num_hidden_layers=1)
|
||||
|
||||
with caplog.at_level("WARNING"):
|
||||
AliceAIForCausalLMConfig.verify_and_update_model_config(
|
||||
SimpleNamespace(hf_config=config)
|
||||
)
|
||||
assert "ignores the MTP configuration and weights" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides,error_type,error",
|
||||
[
|
||||
({}, None, None),
|
||||
(
|
||||
{"kda_allow_negative_eigenvalues": True},
|
||||
NotImplementedError,
|
||||
"kda_allow_negative_eigenvalues=false",
|
||||
),
|
||||
(
|
||||
{"router_score_function": "softmax"},
|
||||
ValueError,
|
||||
"router_score_function='sigmoid'",
|
||||
),
|
||||
(
|
||||
{"router_bias_correction": False},
|
||||
ValueError,
|
||||
"router_bias_correction=true",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_model_config_rejects_unsupported_alice_contract(
|
||||
overrides, error_type, error
|
||||
) -> None:
|
||||
model_config = SimpleNamespace(hf_config=_make_config(**overrides))
|
||||
|
||||
if error is None:
|
||||
AliceAIForCausalLMConfig.verify_and_update_model_config(model_config)
|
||||
else:
|
||||
with pytest.raises(error_type, match=error):
|
||||
AliceAIForCausalLMConfig.verify_and_update_model_config(model_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pipeline_parallel_size,speculative_method,error",
|
||||
[
|
||||
(1, None, None),
|
||||
(2, None, "pipeline_parallel_size=1"),
|
||||
(1, "eagle3", "eagle3"),
|
||||
(1, "dflash", "dflash"),
|
||||
(1, "dspark", "dspark"),
|
||||
(1, "extract_hidden_states", "extract_hidden_states"),
|
||||
],
|
||||
)
|
||||
def test_runtime_config_rejects_unsupported_alice_modes(
|
||||
pipeline_parallel_size, speculative_method, error
|
||||
) -> None:
|
||||
vllm_config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(pipeline_parallel_size=pipeline_parallel_size),
|
||||
speculative_config=(
|
||||
None
|
||||
if speculative_method is None
|
||||
else SimpleNamespace(method=speculative_method)
|
||||
),
|
||||
)
|
||||
|
||||
if error is None:
|
||||
AliceAIForCausalLMConfig.verify_and_update_config(vllm_config)
|
||||
else:
|
||||
with pytest.raises(NotImplementedError, match=error):
|
||||
AliceAIForCausalLMConfig.verify_and_update_config(vllm_config)
|
||||
@@ -1,57 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib.util
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.conftest import VllmRunner
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
importlib.util.find_spec("terratorch") is None,
|
||||
reason="terratorch unavailable while PyPI has `lightning` quarantined; see #41376",
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test() # Hangs otherwise
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
|
||||
"ibm-nasa-geospatial/Prithvi-EO-2.0-300M-BurnScars",
|
||||
],
|
||||
)
|
||||
def test_inference(
|
||||
vllm_runner: type[VllmRunner],
|
||||
model: str,
|
||||
) -> None:
|
||||
pixel_values = torch.full((6, 512, 512), 1.0, dtype=torch.float16)
|
||||
location_coords = torch.full((1, 2), 1.0, dtype=torch.float16)
|
||||
prompt = dict(
|
||||
prompt_token_ids=[1],
|
||||
multi_modal_data={
|
||||
"image": {
|
||||
"pixel_values": pixel_values,
|
||||
"location_coords": location_coords,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
skip_tokenizer_init=True,
|
||||
enable_mm_embeds=True,
|
||||
# Limit the maximum number of sequences to avoid the
|
||||
# test going OOM during the warmup run
|
||||
max_num_seqs=32,
|
||||
default_torch_num_threads=1,
|
||||
) as vllm_model:
|
||||
vllm_output = vllm_model.llm.encode(prompt, pooling_task="plugin")
|
||||
assert torch.equal(
|
||||
torch.isnan(vllm_output[0].outputs.data).any(), torch.tensor(False)
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,611 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import warnings
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config.model import AttnTypeStr, ModelConfig, ModelDType, RunnerOption
|
||||
from vllm.config.pooler import SequencePoolingType, TokenPoolingType
|
||||
from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
|
||||
from vllm.multimodal.processing import InputProcessingContext
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
|
||||
from .. import ci_envs
|
||||
from .registry import HF_EXAMPLE_MODELS
|
||||
|
||||
TokensText = tuple[list[int], str]
|
||||
|
||||
|
||||
def check_outputs_equal(
|
||||
*,
|
||||
outputs_0_lst: Sequence[TokensText],
|
||||
outputs_1_lst: Sequence[TokensText],
|
||||
name_0: str,
|
||||
name_1: str,
|
||||
):
|
||||
"""Compare the two sequences generated by different models,
|
||||
which should be equal.
|
||||
"""
|
||||
assert len(outputs_0_lst) == len(outputs_1_lst)
|
||||
|
||||
for prompt_idx, (outputs_0, outputs_1) in enumerate(
|
||||
zip(outputs_0_lst, outputs_1_lst)
|
||||
):
|
||||
output_ids_0, output_str_0 = outputs_0
|
||||
output_ids_1, output_str_1 = outputs_1
|
||||
|
||||
# The text and token outputs should exactly match
|
||||
fail_msg = (
|
||||
f"Test{prompt_idx}:"
|
||||
f"\n{name_0}:\t{output_str_0!r}"
|
||||
f"\n{name_1}:\t{output_str_1!r}"
|
||||
)
|
||||
|
||||
assert output_str_0 == output_str_1, fail_msg
|
||||
assert output_ids_0 == output_ids_1, fail_msg
|
||||
|
||||
|
||||
# Representation of generated sequence as a tuple of
|
||||
# * Token ID list
|
||||
# * String
|
||||
# * List of top sample logprobs for each sampled token
|
||||
#
|
||||
# Assumes prompt logprobs were not requested.
|
||||
TokensTextLogprobs = tuple[
|
||||
list[int], str, list[dict[int, float]] | SampleLogprobs | None
|
||||
]
|
||||
|
||||
# Allow for tokens to be represented as str's rather than IDs;
|
||||
# tuple of
|
||||
# * Token string representations list
|
||||
# * String
|
||||
# * Optional list of top sample logprobs for each sampled token
|
||||
#
|
||||
# Assumes prompt logprobs were not requested.
|
||||
TextTextLogprobs = tuple[
|
||||
list[str], str, list[dict[str, float]] | list[dict[str, Logprob]] | None
|
||||
]
|
||||
|
||||
# Representation of generated sequence as a tuple of
|
||||
# * Token ID list
|
||||
# * String
|
||||
# * Optional list of top sample logprobs for each sampled token
|
||||
# * Optional list of top prompt logprobs for each prompt token
|
||||
#
|
||||
# Allows prompt logprobs to be requested.
|
||||
TokensTextLogprobsPromptLogprobs = tuple[
|
||||
list[int],
|
||||
str,
|
||||
list[dict[int, float]] | SampleLogprobs | None,
|
||||
list[dict[int, float] | None] | PromptLogprobs | None,
|
||||
]
|
||||
|
||||
|
||||
def check_logprobs_close(
|
||||
*,
|
||||
outputs_0_lst: Sequence[
|
||||
TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs
|
||||
],
|
||||
outputs_1_lst: Sequence[
|
||||
TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs
|
||||
],
|
||||
name_0: str,
|
||||
name_1: str,
|
||||
num_outputs_0_skip_tokens: int = 0,
|
||||
warn_on_mismatch: bool = True,
|
||||
always_check_logprobs: bool = False,
|
||||
) -> None:
|
||||
"""Compare the logprobs of two sequences generated by different models,
|
||||
which should be similar but not necessarily equal.
|
||||
|
||||
How sample logprobs are compared:
|
||||
* `always_check_logprobs == True`: set of highest-logprob token ids
|
||||
must match between seq0 and seq1 at all sampled token offsets
|
||||
* `always_check_logprobs == False`: highest-logprob token ids are
|
||||
only compared at sampled token offsets for which generated token
|
||||
ids don't match
|
||||
|
||||
Prompt logprobs must be provided either for both input sequences, or
|
||||
for neither. If prompt logprobs are provided, then highest-logprob
|
||||
prompt token ids must match between seq0 and seq1 at all prompt token
|
||||
offsets.
|
||||
|
||||
Args:
|
||||
outputs_0_lst: First sequence to compare
|
||||
outputs_0_lst: Second sequence to compare
|
||||
name_0: sequence #0 name
|
||||
name_1: sequence #1 name
|
||||
num_outputs_0_skip_tokens: If > 0, specifies the number of initial
|
||||
sequence #0 tokens & logprobs to discard
|
||||
before comparison, i.e. all
|
||||
of sequence #1 will be compared to
|
||||
sequence #0 beginning at index
|
||||
num_outputs_0_skip_tokens
|
||||
warn_on_mismatch: Issue a warning if there is token-wise or text-wise
|
||||
mismatch between the two sequences
|
||||
always_check_logprobs: If true, check logprobs even when tokens match
|
||||
|
||||
"""
|
||||
assert len(outputs_0_lst) == len(outputs_1_lst)
|
||||
|
||||
# Loop through responses to each prompt.
|
||||
for prompt_idx, (outputs_0, outputs_1) in enumerate(
|
||||
zip(outputs_0_lst, outputs_1_lst)
|
||||
):
|
||||
assert len(outputs_0) == len(outputs_1)
|
||||
if len(outputs_0) == 3:
|
||||
assert len(outputs_1) == 3
|
||||
# Break out tokens, text & sample logprobs
|
||||
# (prompt logprobs were not provided)
|
||||
output_ids_0, output_str_0, logprobs_0 = outputs_0
|
||||
output_ids_1, output_str_1, logprobs_1 = outputs_1
|
||||
elif len(outputs_0) == 4:
|
||||
assert len(outputs_1) == 4
|
||||
# Break out tokens, text, sample logprobs & prompt logprobs
|
||||
(
|
||||
output_ids_0,
|
||||
output_str_0,
|
||||
logprobs_0,
|
||||
prompt_logprobs_0,
|
||||
) = outputs_0
|
||||
(
|
||||
output_ids_1,
|
||||
output_str_1,
|
||||
logprobs_1,
|
||||
prompt_logprobs_1,
|
||||
) = outputs_1
|
||||
|
||||
# Test prompt logprobs closeness
|
||||
if prompt_logprobs_0 is not None and prompt_logprobs_1 is not None:
|
||||
# Both sequences' prompt logprobs lists are not `None`
|
||||
# (although individual list elements may be `None`);
|
||||
# for each token's logprobs:
|
||||
for idx, (logprobs_elem_0, logprobs_elem_1) in enumerate(
|
||||
zip(prompt_logprobs_0, prompt_logprobs_1)
|
||||
):
|
||||
fail_msg = (
|
||||
f"Prompt logprobs test:"
|
||||
f"\n{name_0}:\tPrompt index {idx}\t{logprobs_elem_0}"
|
||||
f"\n{name_1}:\tPrompt index {idx}\t{logprobs_elem_1}"
|
||||
)
|
||||
|
||||
if logprobs_elem_0 is None:
|
||||
# If the seq 0 token's logprobs are `None`,
|
||||
# the seq 1 token's logprobs must be `None`
|
||||
assert logprobs_elem_1 is None, fail_msg
|
||||
else:
|
||||
# If the seq 0 token's logprobs are not `None`,
|
||||
# the seq 1 token's logprobs must not be `None`
|
||||
assert logprobs_elem_1 is not None, fail_msg
|
||||
# Logprobs check: top-k token choices must be the same
|
||||
assert set(logprobs_elem_0.keys()) == set(
|
||||
logprobs_elem_1.keys()
|
||||
), fail_msg
|
||||
else:
|
||||
# Both sequence logprobs lists must be `None`
|
||||
fail_msg = (
|
||||
f"Prompt logprobs test:"
|
||||
f"\n{name_0}:\tlogprobs\t{prompt_logprobs_0}"
|
||||
f"\n{name_1}:\tlogprobs\t{prompt_logprobs_1}"
|
||||
)
|
||||
|
||||
assert prompt_logprobs_0 is None and prompt_logprobs_1 is None, fail_msg
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Outputs tuple must have 3 or 4 elements but "
|
||||
f"{len(outputs_0)} elements were provided: "
|
||||
f"{outputs_0}"
|
||||
)
|
||||
|
||||
if logprobs_0 is None:
|
||||
logprobs_0 = [None] * len(output_ids_0)
|
||||
if logprobs_1 is None:
|
||||
logprobs_1 = [None] * len(output_ids_1)
|
||||
|
||||
# Skip specified number of initial sequence #0 tokens
|
||||
# & logprobs, leaving output text as-is for simplicity
|
||||
# (text mismatches may generate warnings but do not
|
||||
# cause the test to fail.)
|
||||
if num_outputs_0_skip_tokens < 0:
|
||||
raise ValueError("num_outputs_0_skip_tokens must be non-negative")
|
||||
output_ids_0 = output_ids_0[num_outputs_0_skip_tokens:]
|
||||
logprobs_0 = logprobs_0[num_outputs_0_skip_tokens:]
|
||||
|
||||
# Loop through generated tokens.
|
||||
for idx, (output_id_0, output_id_1) in enumerate(
|
||||
zip(output_ids_0, output_ids_1)
|
||||
):
|
||||
is_tok_mismatch = output_id_0 != output_id_1
|
||||
|
||||
# If generated tokens don't match
|
||||
# or it is desired to always check logprobs,
|
||||
# then
|
||||
if is_tok_mismatch or always_check_logprobs:
|
||||
logprobs_elem_0 = logprobs_0[idx]
|
||||
logprobs_elem_1 = logprobs_1[idx]
|
||||
|
||||
# Each predicted token must be in top N logprobs of the other
|
||||
fail_msg = (
|
||||
f"Test{prompt_idx}:"
|
||||
f"\nMatched tokens:\t{output_ids_0[:idx]}"
|
||||
f"\n{name_0}:\t{output_str_0!r}\t{logprobs_elem_0}"
|
||||
f"\n{name_1}:\t{output_str_1!r}\t{logprobs_elem_1}"
|
||||
)
|
||||
|
||||
assert logprobs_elem_0 is not None, fail_msg
|
||||
assert logprobs_elem_1 is not None, fail_msg
|
||||
assert output_id_0 in logprobs_elem_1, fail_msg
|
||||
assert output_id_1 in logprobs_elem_0, fail_msg
|
||||
|
||||
if warn_on_mismatch and is_tok_mismatch:
|
||||
with warnings.catch_warnings():
|
||||
# This ensures that repeated warnings are shown
|
||||
# in the output, not just the first occurrence
|
||||
warnings.simplefilter("always")
|
||||
|
||||
warnings.warn(fail_msg, stacklevel=2)
|
||||
|
||||
# Break out since sequences will now diverge.
|
||||
break
|
||||
else:
|
||||
if output_str_0 != output_str_1 and warn_on_mismatch:
|
||||
# The token outputs exactly match,
|
||||
# so the text outputs should exactly match as well
|
||||
fail_msg = (
|
||||
f"Test{prompt_idx}:"
|
||||
f"\n{name_0}:\t{output_str_0!r}"
|
||||
f"\n{name_1}:\t{output_str_1!r}"
|
||||
)
|
||||
|
||||
with warnings.catch_warnings():
|
||||
# This ensures that repeated warnings are shown
|
||||
# in the output, not just the first occurrence
|
||||
warnings.simplefilter("always")
|
||||
|
||||
warnings.warn(fail_msg, stacklevel=2)
|
||||
|
||||
|
||||
def build_model_context(
|
||||
model_id: str,
|
||||
runner: RunnerOption = "auto",
|
||||
dtype: ModelDType = "auto",
|
||||
model_config_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]] | None = None,
|
||||
mm_processor_cache_gb: int = 0,
|
||||
):
|
||||
"""Creates an InputProcessingContext for a given model.
|
||||
|
||||
Args:
|
||||
model_id: ID of the model being considered.
|
||||
mm_processor_kwargs: optional processor kwargs for to be leveraged
|
||||
in the input processor, mapper, dummy data creation, etc.
|
||||
limit_mm_per_prompt: Multimodal limits.
|
||||
|
||||
Returns:
|
||||
InputProcessingContext for the model being considered.
|
||||
|
||||
"""
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(
|
||||
on_fail="skip",
|
||||
check_max_version=False,
|
||||
check_version_reason="vllm",
|
||||
)
|
||||
|
||||
model_config_kwargs = model_config_kwargs or {}
|
||||
limit_mm_per_prompt = {
|
||||
modality: dict(limit) if isinstance(limit, Mapping) else limit
|
||||
for modality, limit in (limit_mm_per_prompt or {}).items()
|
||||
}
|
||||
model_config = ModelConfig(
|
||||
model_id,
|
||||
runner=runner,
|
||||
tokenizer=model_info.tokenizer or model_id,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
revision=model_info.revision,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
dtype=dtype,
|
||||
seed=0,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
mm_processor_cache_gb=mm_processor_cache_gb,
|
||||
hf_overrides=model_info.hf_overrides,
|
||||
skip_tokenizer_init=model_info.require_embed_inputs,
|
||||
enable_prompt_embeds=model_info.require_embed_inputs,
|
||||
enable_mm_embeds=model_info.require_embed_inputs,
|
||||
enforce_eager=model_info.enforce_eager,
|
||||
**model_config_kwargs,
|
||||
)
|
||||
|
||||
return InputProcessingContext(
|
||||
model_config,
|
||||
tokenizer=cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
|
||||
|
||||
def check_embeddings_close(
|
||||
*,
|
||||
embeddings_0_lst: Sequence[list[float]],
|
||||
embeddings_1_lst: Sequence[list[float]],
|
||||
name_0: str,
|
||||
name_1: str,
|
||||
tol: float = 1e-3,
|
||||
) -> None:
|
||||
assert len(embeddings_0_lst) == len(embeddings_1_lst)
|
||||
|
||||
for prompt_idx, (embeddings_0, embeddings_1) in enumerate(
|
||||
zip(embeddings_0_lst, embeddings_1_lst)
|
||||
):
|
||||
assert len(embeddings_0) == len(embeddings_1), (
|
||||
f"Length mismatch: {len(embeddings_0)} vs. {len(embeddings_1)}"
|
||||
)
|
||||
|
||||
sim = F.cosine_similarity(
|
||||
torch.tensor(embeddings_0), torch.tensor(embeddings_1), dim=0
|
||||
)
|
||||
|
||||
fail_msg = (
|
||||
f"Test{prompt_idx}:"
|
||||
f"\nCosine similarity: \t{sim:.4f}"
|
||||
f"\n{name_0}:\t{embeddings_0[:16]!r}"
|
||||
f"\n{name_1}:\t{embeddings_1[:16]!r}"
|
||||
)
|
||||
|
||||
assert sim >= 1 - tol, fail_msg
|
||||
|
||||
|
||||
def matryoshka_fy(tensor: torch.Tensor, dimensions: int):
|
||||
tensor = torch.tensor(tensor)
|
||||
tensor = tensor[..., :dimensions]
|
||||
tensor = F.normalize(tensor, p=2, dim=1)
|
||||
return tensor
|
||||
|
||||
|
||||
def softmax(data):
|
||||
if data.shape[-1] == 1:
|
||||
return F.sigmoid(data)
|
||||
else:
|
||||
return F.softmax(data, dim=-1)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelInfo:
|
||||
name: str
|
||||
revision: str | None = None
|
||||
architecture: str = ""
|
||||
dtype: str = "auto"
|
||||
max_model_len: int | None = None
|
||||
hf_dtype: str = "float32"
|
||||
hf_overrides: dict[str, Any] | None = None
|
||||
seq_pooling_type: SequencePoolingType | None = None
|
||||
tok_pooling_type: TokenPoolingType | None = None
|
||||
attn_type: AttnTypeStr | None = None
|
||||
is_prefix_caching_supported: bool | None = None
|
||||
is_chunked_prefill_supported: bool | None = None
|
||||
enable_test: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class EmbedModelInfo(ModelInfo):
|
||||
mteb_score: float | None = None
|
||||
# Per-model MTEB score tolerance; falls back to MTEB_EMBED_TOL.
|
||||
mteb_tol: float | None = None
|
||||
is_matryoshka: bool = False
|
||||
matryoshka_dimensions: list[int] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RerankModelInfo(ModelInfo):
|
||||
mteb_score: float | None = None
|
||||
# Per-model MTEB score tolerance; falls back to MTEB_RERANK_TOL.
|
||||
mteb_tol: float | None = None
|
||||
chat_template_name: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class GenerateModelInfo(ModelInfo):
|
||||
hf_dtype: str = "auto"
|
||||
hf_ppl: float | None = None
|
||||
|
||||
|
||||
def get_vllm_extra_kwargs(model_info: ModelInfo, vllm_extra_kwargs):
|
||||
# A model family has many models with the same architecture,
|
||||
# and we don't need to test each one.
|
||||
if not ci_envs.VLLM_CI_NO_SKIP and not model_info.enable_test:
|
||||
import pytest
|
||||
|
||||
pytest.skip("Skipping test.")
|
||||
|
||||
# Allow vllm to test using the given dtype, such as float32
|
||||
vllm_extra_kwargs = vllm_extra_kwargs or {}
|
||||
vllm_extra_kwargs["dtype"] = ci_envs.VLLM_CI_DTYPE or model_info.dtype
|
||||
|
||||
# Allow vllm to test using hf_overrides
|
||||
if model_info.hf_overrides is not None:
|
||||
vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides
|
||||
|
||||
# Allow changing the head dtype used by vllm in tests
|
||||
if ci_envs.VLLM_CI_HEAD_DTYPE is not None:
|
||||
if "hf_overrides" not in vllm_extra_kwargs:
|
||||
vllm_extra_kwargs["hf_overrides"] = {}
|
||||
vllm_extra_kwargs["hf_overrides"]["head_dtype"] = ci_envs.VLLM_CI_HEAD_DTYPE
|
||||
|
||||
# Allow control over whether tests use enforce_eager
|
||||
if ci_envs.VLLM_CI_ENFORCE_EAGER is not None:
|
||||
vllm_extra_kwargs["enforce_eager"] = ci_envs.VLLM_CI_ENFORCE_EAGER
|
||||
|
||||
return vllm_extra_kwargs
|
||||
|
||||
|
||||
def dummy_hf_overrides(
|
||||
hf_config: PreTrainedConfig,
|
||||
*,
|
||||
model_arch: str = "",
|
||||
exist_overrides: dict[str, Any] | None = None,
|
||||
use_original_num_layers: bool = False,
|
||||
) -> PreTrainedConfig:
|
||||
"""Dummy HF overrides function used to create dummy model
|
||||
with only minimum nums of layer.
|
||||
"""
|
||||
# Copy because this helper is called more than once
|
||||
# while loading config, and we `.pop()`
|
||||
exist_overrides = (exist_overrides or {}).copy()
|
||||
text_config_override = exist_overrides.pop("text_config", None)
|
||||
hf_config.update(exist_overrides)
|
||||
|
||||
text_config = hf_config.get_text_config()
|
||||
if text_config_override is not None:
|
||||
# multimodal test models may override *some* text-model fields
|
||||
text_config.update(text_config_override)
|
||||
|
||||
# Ensure at least 2 expert per group
|
||||
# Since `grouped_topk` assumes top-2
|
||||
n_group = getattr(text_config, "n_group", None)
|
||||
# Kimi uses `num_expert_group` instead of `n_group`.
|
||||
if n_group is None:
|
||||
n_group = getattr(text_config, "num_expert_group", None)
|
||||
# InternS1Pro uses `router_n_groups` instead of `n_group`.
|
||||
if n_group is None:
|
||||
n_group = getattr(text_config, "router_n_groups", None)
|
||||
num_experts = n_group * 2 if n_group is not None else 2
|
||||
|
||||
# we use three layers for Gemma-3n to check
|
||||
# both normal layer and kv_shared_layer
|
||||
if use_original_num_layers:
|
||||
# Use the original number of layers from the config
|
||||
num_layers = getattr(text_config, "num_layers", 1)
|
||||
num_hidden_layers = getattr(text_config, "num_hidden_layers", 1)
|
||||
else:
|
||||
# Use minimal layers for testing
|
||||
num_layers = 1
|
||||
num_hidden_layers = (
|
||||
3
|
||||
if model_arch
|
||||
in (
|
||||
"Gemma3nForConditionalGeneration",
|
||||
"Gemma4ForCausalLM",
|
||||
"Gemma4ForConditionalGeneration",
|
||||
"Gemma4MTPModel",
|
||||
"DiffusionGemmaForBlockDiffusion",
|
||||
)
|
||||
else 1
|
||||
)
|
||||
|
||||
update_dict = {
|
||||
"num_layers": num_layers,
|
||||
# For Gemma-3n
|
||||
"num_kv_shared_layers": 1,
|
||||
}
|
||||
|
||||
_hf_config = hf_config
|
||||
|
||||
class DummyConfig:
|
||||
hf_config = _hf_config
|
||||
hf_text_config = text_config
|
||||
|
||||
# Keep architecture conversion on the default multimodal path; this
|
||||
# helper only needs HF-derived fields, not deployment MM limits.
|
||||
@staticmethod
|
||||
def _supports_multimodal_for_mm_prefix() -> bool:
|
||||
return True
|
||||
|
||||
model_arch_config = ModelConfig.get_model_arch_config(DummyConfig)
|
||||
# Only set MoE related config when the model has MoE layers.
|
||||
# Otherwise all models detected as MoE by _get_transformers_backend_cls.
|
||||
if model_arch_config.num_experts > 0:
|
||||
num_experts_per_tok = 2
|
||||
if model_arch in (
|
||||
"Llama4ForConditionalGeneration",
|
||||
"Llama4ForCausalLM",
|
||||
"EagleLlama4ForCausalLM",
|
||||
):
|
||||
num_experts_per_tok = 1
|
||||
elif model_arch == "InternS1ProForConditionalGeneration":
|
||||
assert n_group is not None
|
||||
num_experts_per_tok = n_group
|
||||
update_dict.update(
|
||||
{
|
||||
"num_experts": num_experts,
|
||||
"num_experts_per_tok": num_experts_per_tok,
|
||||
# Kimi uses `num_experts_per_token`.
|
||||
"num_experts_per_token": num_experts_per_tok,
|
||||
"num_local_experts": num_experts,
|
||||
# Otherwise there will not be any expert layers
|
||||
"first_k_dense_replace": 0,
|
||||
# To avoid OOM on DeepSeek-V3
|
||||
"n_routed_experts": num_experts,
|
||||
}
|
||||
)
|
||||
|
||||
# Update num_hidden_layers for non-Longcat architectures (Longcat derives it
|
||||
# from num_layers for its dual-attention layers).
|
||||
if model_arch not in (
|
||||
"LongcatFlashForCausalLM",
|
||||
"LongCatFlashMTPModel",
|
||||
"LongcatFlashNgramForCausalLM",
|
||||
):
|
||||
update_dict["num_hidden_layers"] = num_hidden_layers
|
||||
|
||||
text_config.update(update_dict)
|
||||
|
||||
# Keep per-layer metadata consistent with the reduced layer count.
|
||||
# HY V4 uses this to decide which sparse-attention indexers are shared.
|
||||
if indexer_types := getattr(text_config, "indexer_types", None):
|
||||
text_config.update({"indexer_types": indexer_types[:num_hidden_layers]})
|
||||
|
||||
# Update n_layers and moe configs for Moondream3 model
|
||||
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
|
||||
text_config.update(
|
||||
{
|
||||
"n_layers": num_hidden_layers,
|
||||
"moe_num_experts": num_experts,
|
||||
"moe_experts_per_token": 2,
|
||||
"moe_start_layer": num_hidden_layers,
|
||||
}
|
||||
)
|
||||
|
||||
if hasattr(hf_config, "vision_config"):
|
||||
vision_config = hf_config.vision_config
|
||||
vision_config.update(
|
||||
{
|
||||
"num_layers": 1,
|
||||
"num_hidden_layers": 1,
|
||||
}
|
||||
)
|
||||
|
||||
# Keep per-layer metadata consistent with the reduced layer count.
|
||||
if layer_types := getattr(vision_config, "layer_types", None):
|
||||
vision_config.update({"layer_types": layer_types[:1]})
|
||||
|
||||
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
|
||||
vision_config.update({"enc_n_layers": 1})
|
||||
|
||||
# e.g.: ibm-granite/granite-speech-3.3-2b
|
||||
if hasattr(hf_config, "encoder_config"):
|
||||
hf_config.encoder_config.update(
|
||||
{
|
||||
"num_layers": 1,
|
||||
"num_hidden_layers": 1,
|
||||
}
|
||||
)
|
||||
|
||||
# e.g.: Qwen/Qwen2-Audio-7B-Instruct
|
||||
# audio_config may exist but be None (e.g. audio-less Gemma4 variants).
|
||||
if getattr(hf_config, "audio_config", None) is not None:
|
||||
hf_config.audio_config.update(
|
||||
{
|
||||
"num_layers": 1,
|
||||
"num_hidden_layers": 1,
|
||||
"encoder_layers": 1,
|
||||
}
|
||||
)
|
||||
|
||||
return hf_config
|
||||
Reference in New Issue
Block a user