[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
@@ -1,240 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import cast
import numpy as np
import pytest
import torch
from vllm.config import CompilationConfig, VllmConfig
from vllm.config.compilation import CompilationMode
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models import gemma
from vllm.model_executor.models.gemma3n import (
Gemma3nTextModel,
_kv_sharing_weights_mapper,
)
from vllm.model_executor.models.gemma4 import (
Gemma4ForCausalLM,
_gemma4_layer_weights_mapper,
)
from vllm.model_executor.models.gemma4_dspark import (
Gemma4DSparkForCausalLM,
Gemma4DSparkModel,
)
MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"]
@pytest.mark.usefixtures("dist_init")
@pytest.mark.parametrize(
"enabled,has_weights,with_markov",
[
pytest.param(True, True, False, id="hidden-only"),
pytest.param(True, True, True, id="with-markov"),
pytest.param(True, False, True, id="missing-weights"),
pytest.param(False, True, True, id="disabled-head"),
],
)
def test_gemma4_dspark_loads_confidence_head(
monkeypatch, enabled, has_weights, with_markov
) -> None:
"""Use checkpoint confidence parameters, or disable an unavailable head."""
config = SimpleNamespace(
vocab_size=64,
hidden_size=8,
target_layer_ids=[0, 1],
num_hidden_layers=0,
rms_norm_eps=1e-6,
markov_rank=4,
enable_confidence_head=enabled,
confidence_head_with_markov=with_markov,
)
vllm_config = SimpleNamespace(
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
model_config=SimpleNamespace(dtype=torch.bfloat16),
speculative_config=SimpleNamespace(
draft_model_config=SimpleNamespace(hf_config=config),
),
)
monkeypatch.setattr(Gemma4DSparkModel, "_build_fused_kv_buffers", lambda _: None)
model = Gemma4DSparkForCausalLM(vllm_config=cast(VllmConfig, vllm_config))
width = config.hidden_size + (config.markov_rank if with_markov else 0)
weight = torch.arange(width, dtype=torch.bfloat16).reshape(1, -1) / 16
bias = torch.tensor([-0.5], dtype=torch.bfloat16)
loaded = model.load_weights(
[("confidence_head.proj.weight", weight), ("confidence_head.proj.bias", bias)]
if has_weights
else []
)
if not (enabled and has_weights):
assert model.model.confidence_head is None
assert not loaded
return
assert loaded == {
"model.confidence_head.proj.weight",
"model.confidence_head.proj.bias",
}
assert model.model.confidence_head.proj.weight.dtype == torch.float32
hidden = torch.full((3, config.hidden_size), 0.25, dtype=torch.bfloat16)
markov = torch.full((3, config.markov_rank), -0.5, dtype=torch.bfloat16)
inputs = torch.cat([hidden, markov], dim=-1) if with_markov else hidden
expected = (inputs.float() @ weight.float().T + bias.float()).sigmoid().squeeze(-1)
torch.testing.assert_close(model.compute_confidence(hidden, markov), expected)
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
"""A physical LM head must load after checkpoint-driven untying."""
class StubGemmaModel(torch.nn.Module):
def __init__(self, *, vllm_config, prefix):
super().__init__()
self.embed_tokens = VocabParallelEmbedding(4, 2)
self.make_empty_intermediate_tensors = None
monkeypatch.setattr(gemma, "GemmaModel", StubGemmaModel)
config = SimpleNamespace(
vocab_size=4,
hidden_size=2,
tie_word_embeddings=False,
)
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(hf_config=config),
quant_config=None,
)
model = gemma.GemmaForCausalLM(vllm_config=cast(VllmConfig, vllm_config))
embedding_weight = torch.full((4, 2), 1.0)
lm_head_weight = torch.full((4, 2), 2.0)
loaded = model.load_weights(
[
("model.embed_tokens.weight", embedding_weight),
("lm_head.weight", lm_head_weight),
]
)
assert loaded == {"model.embed_tokens.weight", "lm_head.weight"}
assert torch.equal(model.model.embed_tokens.weight[:4], embedding_weight)
assert torch.equal(model.lm_head.weight[:4], lm_head_weight)
@pytest.mark.cpu_test
def test_gemma4_attention_mapper() -> None:
"""Layers with a qkv_proj pack q/k/v; `attention_k_eq_v` full-attention
layers also load K as the V shard, and leave any v_proj such a checkpoint
ships unmapped so it fails the load rather than silently overwriting V;
KV-shared layers keep q_proj and drop the K/V tensors original checkpoints
still ship for them."""
config = SimpleNamespace(
num_hidden_layers=3,
num_kv_shared_layers=1,
attention_k_eq_v=True,
layer_types=["sliding_attention", "full_attention", "sliding_attention"],
)
weights = [
(f"model.layers.{i}.self_attn.{tensor}.weight", torch.full((2, 2), i + 1.0))
for i in range(3)
for tensor in ("q_proj", "k_proj", "k_norm")
] + [
("model.layers.0.mlp.up_proj.weight", torch.empty(0)),
("model.layers.1.self_attn.v_proj.weight", torch.empty(0)),
]
mapper = _gemma4_layer_weights_mapper(config)
mapped = list(mapper.apply(weights))
assert [(name, getattr(w, "shard_id", None)) for name, w in mapped] == [
("model.layers.0.self_attn.qkv_proj.weight", "q"),
("model.layers.0.self_attn.qkv_proj.weight", "k"),
("model.layers.0.self_attn.k_norm.weight", None),
("model.layers.1.self_attn.qkv_proj.weight", "q"),
("model.layers.1.self_attn.qkv_proj.weight", "k"),
("model.layers.1.self_attn.qkv_proj.weight", "v"),
("model.layers.1.self_attn.k_norm.weight", None),
("model.layers.2.self_attn.q_proj.weight", None),
("model.layers.0.mlp.gate_up_proj.weight", 1),
("model.layers.1.self_attn.v_proj.weight", None),
]
k_weight, v_weight = weights[4][1], mapped[5][1]
assert torch.equal(v_weight, k_weight) and v_weight is not k_weight
@pytest.mark.cpu_test
def test_gemma4_expert_names_strip_language_model_prefix() -> None:
"""The text-only path reuses the conditional wrapper's checkpoint naming,
so fused and per-expert tensors reach the experts under `model.*`."""
prefix = "model.language_model.layers.0."
weights = [
(prefix + name, torch.empty(0))
for name in (
"experts.gate_up_proj",
"experts.3.down_proj.weight_packed",
"router.per_expert_scale",
)
]
mapped = [
(name, getattr(w, "shard_id", None))
for name, w in Gemma4ForCausalLM.hf_to_vllm_mapper.apply(weights)
]
assert mapped == [
("model.layers.0.experts.gate_up_proj", None),
("model.layers.0.experts.3.down_proj.weight_packed", None),
("model.layers.0.router.per_expert_scale", None),
]
@pytest.mark.cpu_test
def test_gemma3n_kv_shared_layer_mapper() -> None:
"""Only non-shared layers pack q/k/v into qkv_proj; KV-shared layers keep
q_proj and drop the redundant K/V tensors original checkpoints ship."""
config = SimpleNamespace(num_hidden_layers=4, num_kv_shared_layers=2)
mapper = Gemma3nTextModel.hf_to_vllm_mapper | _kv_sharing_weights_mapper(config)
weights = [
(f"layers.{i}.self_attn.{tensor}.weight", torch.empty(0))
for i in (1, 3)
for tensor in ("q_proj", "k_proj", "v_proj", "k_norm", "o_proj")
]
mapped = [
(name, getattr(weight, "shard_id", None))
for name, weight in mapper.apply(weights)
]
assert mapped == [
("layers.1.self_attn.qkv_proj.weight", "q"),
("layers.1.self_attn.qkv_proj.weight", "k"),
("layers.1.self_attn.qkv_proj.weight", "v"),
("layers.1.self_attn.k_norm.weight", None),
("layers.1.self_attn.o_proj.weight", None),
("layers.3.self_attn.q_proj.weight", None),
("layers.3.self_attn.o_proj.weight", None),
]
@pytest.mark.parametrize("model", MODELS)
def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None:
with monkeypatch.context() as m:
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
with vllm_runner(
model,
load_format="dummy",
) as llm:
if model == "google/gemma-3-4b-it":
normalizers = llm.llm.collective_rpc(
lambda self: (
self.model_runner.model.language_model.model.normalizer.cpu().item()
) # noqa: E501
)
config = llm.llm.llm_engine.model_config.hf_config.text_config
else:
normalizers = llm.llm.collective_rpc(
lambda self: self.model_runner.model.model.normalizer.cpu().item()
)
config = llm.llm.llm_engine.model_config.hf_config
assert np.allclose(normalizers, config.hidden_size**0.5, rtol=2e-3)
@@ -1,78 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import GraniteConfig
from vllm.model_executor.models.granite import granite_layer_attn_params
from ...utils import check_logprobs_close
MODELS = [
# TODO(sang): Sliding window should be tested separately.
"ibm/PowerLM-3b",
"ibm/PowerMoE-3b",
"ibm-granite/granite-swash-2b",
"ibm-granite/granite-swash-3b-a600m",
]
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.cpu_model
def test_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
max_tokens: int,
num_logprobs: int,
) -> None:
with hf_runner(model, dtype=dtype) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
example_prompts, max_tokens, num_logprobs
)
with vllm_runner(model, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
def test_granite_swa_features_are_off_without_swa_config():
"""A plain Granite config must not pick up sliding windows or sinks."""
config = GraniteConfig(num_hidden_layers=4)
theta = config.rope_parameters["rope_theta"]
assert [granite_layer_attn_params(config, i) for i in range(4)] == [
(None, theta, False)
] * 4
@pytest.mark.parametrize(
"attention_sinks, expected_sink", [(None, True), (False, False)]
)
def test_granite_swa_features_resolve_per_layer(attention_sinks, expected_sink):
"""`layer_types`/`layer_rope_theta` apply per layer; theta 0 means NoPE."""
kwargs = {} if attention_sinks is None else {"attention_sinks": attention_sinks}
config = GraniteConfig(
num_hidden_layers=3,
sliding_window=128,
layer_types=["full_attention", "sliding_attention", "sliding_attention"],
layer_rope_theta=[10000.0, 0.0, 1000000.0],
**kwargs,
)
assert [granite_layer_attn_params(config, i) for i in range(3)] == [
(None, 10000.0, expected_sink),
(128, 0.0, expected_sink),
(128, 1000000.0, expected_sink),
]
@@ -1,109 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from ....utils import large_gpu_test
from ...utils import check_logprobs_close
MODELS = [
"microsoft/Phi-3.5-MoE-instruct",
]
def test_phimoe_routing_function():
from vllm.model_executor.models.phimoe import phimoe_routing_function
test_case = {
0: {
"hidden_states": torch.tensor(
[1, 2, 3, 4, 5, 6, 7, 8], dtype=torch.float32, requires_grad=False
).view(4, 2),
"gating_output": torch.tensor(
[0.1, 0.2, 0.3, 0.4], dtype=torch.float32, requires_grad=False
),
"topk": 2,
"renormalize": False,
},
1: {
"hidden_states": torch.tensor(
[1, 2, 3, 4, 5, 6, 7, 8], dtype=torch.float32, requires_grad=False
).view(4, 2),
"gating_output": torch.tensor(
[0.4, 0.2, 0.3, 0.4], dtype=torch.float32, requires_grad=False
),
"topk": 2,
"renormalize": False,
},
}
ground_truth = {
0: {
"topk_weights": torch.tensor(
[1.0, 1.0], dtype=torch.float32, requires_grad=False
),
"topk_ids": torch.tensor([3, 2], dtype=torch.long, requires_grad=False),
},
1: {
"topk_weights": torch.tensor(
[0.5, 1.0], dtype=torch.float32, requires_grad=False
),
"topk_ids": torch.tensor([0, 3], dtype=torch.long, requires_grad=False),
},
}
for test_id in test_case:
topk_weights, topk_ids = phimoe_routing_function(**test_case[test_id])
assert torch.allclose(topk_weights, ground_truth[test_id]["topk_weights"])
assert torch.equal(topk_ids, ground_truth[test_id]["topk_ids"])
# There is a known issue that triggers `AttributeError: 'DynamicCache'
# object has no attribute 'seen_tokens'` when running:
# `tests/models/language/generation/test_phimoe.py::test_models
# [5-64-bfloat16-microsoft/Phi-3.5-MoE-instruct]`
# This issue is being investigated and tracked in:
# https://huggingface.co/microsoft/Phi-3.5-MoE-instruct/discussions/58
# It is platform-agnostic. Therefore, we skip this test on all platforms for now.
@pytest.mark.skip(
reason="Skipping due to known issue: "
"'DynamicCache' object has no attribute 'seen_tokens'. See: "
"https://huggingface.co/microsoft/Phi-3.5-MoE-instruct/discussions/58 "
"for details.",
)
@pytest.mark.skipif(
condition=current_platform.is_cpu(),
reason="This test takes a lot time to run on CPU, "
"and vllm CI's disk space is not enough for this model.",
)
@large_gpu_test(min_gb=80)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("num_logprobs", [5])
def test_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
max_tokens: int,
num_logprobs: int,
) -> None:
with hf_runner(model, dtype=dtype) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
example_prompts, max_tokens, num_logprobs
)
with vllm_runner(model, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
-43
View File
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pytest configuration for vLLM language generation tests."""
import gc
import pytest
import torch
from tests.utils import wait_for_memory_to_settle
from vllm.logger import init_logger
from vllm.platforms import current_platform
logger = init_logger(__name__)
def pytest_sessionstart(session):
"""Configure ROCm-specific settings before test session starts."""
if current_platform.is_rocm():
torch.set_float32_matmul_precision("high")
@pytest.fixture(autouse=True)
def release_gpu_memory_between_tests():
"""Reclaim cached GPU memory between tests in this shard.
These tests start many vLLM/HF engines sequentially in one pytest
process; without explicit reclamation between tests, VRAM fragmentation
accumulates and later tests fail at engine startup ("Engine core
initialization failed").
"""
yield
gc.collect()
if torch.accelerator.is_available():
torch.accelerator.empty_cache()
if current_platform.is_rocm():
return
try:
wait_for_memory_to_settle()
except ValueError as e:
# Longer-lived (class/module-scoped) engine fixtures may legitimately
# still hold VRAM; nothing further to reclaim in that case.
logger.info("Failed to clean GPU memory: %s", e)
@@ -1,90 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
import openai
import pytest
from tests.conftest import HfRunner
from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
def float_embeddings(response: EmbeddingResponse) -> list[list[float]]:
embeddings = []
for d in response.data:
assert isinstance(d.embedding, list)
embeddings.append(d.embedding)
return embeddings
def run_embedding_correctness_test(
hf_model: "HfRunner",
inputs: list[str],
vllm_outputs: Sequence[list[float]],
dimensions: int | None = None,
):
hf_outputs = hf_model.encode(inputs)
if dimensions:
hf_outputs = matryoshka_fy(hf_outputs, dimensions)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
def correctness_test_embed_models(
hf_runner,
vllm_runner,
model_info: EmbedModelInfo,
example_prompts,
vllm_extra_kwargs=None,
hf_model_callback=None,
):
pytest.skip("Debug only, ci prefers to use mteb test.")
# The example_prompts has ending "\n", for example:
# "Write a short story about a robot that dreams for the first time.\n"
# sentence_transformers will strip the input texts, see:
# https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
# This makes the input_ids different between hf_model and vllm_model.
# So we need to strip the input texts to avoid test failing.
example_prompts = [str(s).strip() for s in example_prompts]
vllm_extra_kwargs = vllm_extra_kwargs or {}
vllm_extra_kwargs["dtype"] = model_info.dtype
if model_info.hf_overrides is not None:
vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides
with vllm_runner(
model_info.name, runner="pooling", max_model_len=None, **vllm_extra_kwargs
) as vllm_model:
vllm_outputs = vllm_model.embed(example_prompts)
with hf_runner(
model_info.name,
dtype=model_info.hf_dtype,
is_sentence_transformer=True,
) as hf_model:
if hf_model_callback is not None:
hf_model_callback(hf_model)
run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
async def run_client_embeddings(
client: openai.AsyncOpenAI,
model_name: str,
queries: list[str],
instruction: str = "",
) -> list[list[float]]:
outputs = await client.embeddings.create(
model=model_name,
input=[instruction + q for q in queries],
)
return [data.embedding for data in outputs.data]
@@ -1,151 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModel
from tests.models.utils import check_embeddings_close
from tests.utils import VLLM_PATH
from vllm import TokensPrompt
from vllm.config import PoolerConfig
@pytest.mark.parametrize(
"model",
["Qwen/Qwen3-Embedding-0.6B"],
)
@torch.inference_mode
def test_decoder_token_embed_model_runner_v2(
hf_runner, vllm_runner, monkeypatch, model: str
):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
chunk_size = 10
n_prompt_tokens = [55, 56, 57]
token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
with vllm_runner(
model,
runner="pooling",
pooler_config=PoolerConfig(task="token_embed"),
max_model_len=128,
max_num_batched_tokens=chunk_size,
enforce_eager=True,
# `enable_chunked_prefill`: Set to `False` instead of `None` in VllmRunner
enable_chunked_prefill=True,
enable_prefix_caching=True,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = vllm_model.token_embed(
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
)
with hf_runner(
model,
auto_cls=AutoModel,
) as hf_model:
hf_outputs = []
for token_prompt in token_prompts:
inputs = hf_model.wrap_device({"input_ids": torch.tensor([token_prompt])})
input_ids = inputs["input_ids"]
output = hf_model.model(input_ids)
hf_outputs.append(output.last_hidden_state.cpu().float()[0])
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
check_embeddings_close(
embeddings_0_lst=hf_output,
embeddings_1_lst=vllm_output,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
_RERANKER_HF_OVERRIDES = {
"architectures": ["Qwen3ForSequenceClassification"],
"classifier_from_token": ["no", "yes"],
"is_original_qwen3_reranker": True,
}
_RERANKER_TEMPLATE = VLLM_PATH / "examples/pooling/score/template/qwen3_reranker.jinja"
@pytest.mark.parametrize("model", ["Qwen/Qwen3-Reranker-0.6B"])
@torch.inference_mode
def test_last_pool_score_chunked_prefill_matches_unchunked(vllm_runner, model: str):
"""LAST-pooling score must not depend on whether the prompt is chunked.
Regression test for a wrong-score bug where, under ``torch.compile`` (i.e.
not ``enforce_eager``), a query+document pair long enough to be split across
prefill chunks produced a corrupted last-token hidden state and thus a
wrong relevance score, while the same input scored correctly unchunked.
The existing pooling+chunked-prefill coverage runs ``enforce_eager=True``
and so never exercised the compiled path.
"""
chat_template = _RERANKER_TEMPLATE.read_text()
query = "What organelle produces energy in the cell?"
# A long, matching document so query + doc exceeds the chunk size below.
document = (
"The mitochondria is the powerhouse of the cell. It generates most of "
"the cell chemical energy through oxidative phosphorylation. "
) * 400
def score_with(max_num_batched_tokens: int) -> float:
with vllm_runner(
model,
runner="pooling",
hf_overrides=_RERANKER_HF_OVERRIDES,
max_model_len=16384,
max_num_batched_tokens=max_num_batched_tokens,
enable_chunked_prefill=True,
enable_prefix_caching=False,
) as vllm_model:
return vllm_model.score(query, document, chat_template=chat_template)[0]
# Chunk size forces the prompt across several prefill chunks; the large
# budget keeps it in a single chunk as the reference.
chunked = score_with(2048)
unchunked = score_with(16384)
assert chunked == pytest.approx(unchunked, abs=5e-2), (
f"chunked score {chunked} diverged from unchunked {unchunked}"
)
@torch.inference_mode
def test_sequence_embed_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
model = "Qwen/Qwen3-Embedding-0.6B"
chunk_size = 10
token_prompts = [[1024 + i for i in range(n)] for n in (25, 27)]
prompts = [TokensPrompt(prompt_token_ids=t) for t in token_prompts]
with hf_runner(model, auto_cls=AutoModel) as hf_model:
hf_outputs = []
for token_prompt in token_prompts:
inputs = hf_model.wrap_device({"input_ids": torch.tensor([token_prompt])})
output = hf_model.model(inputs["input_ids"])
embedding = torch.nn.functional.normalize(
output.last_hidden_state.float()[0, -1], dim=0
)
hf_outputs.append(embedding.cpu().tolist())
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
runner="pooling",
pooler_config=PoolerConfig(task="embed"),
max_model_len=64,
max_num_batched_tokens=chunk_size,
max_num_seqs=2,
gpu_memory_utilization=0.25,
enforce_eager=True,
enable_chunked_prefill=True,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = vllm_model.embed(prompts)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
@@ -1,113 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModelForSequenceClassification
from tests.models.language.pooling.embed_utils import run_embedding_correctness_test
@pytest.mark.parametrize(
"model",
["jason9693/Qwen2.5-1.5B-apeach"],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_classify_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
# example_prompts is too short for testing prefix_caching
example_prompts = [s * 10 for s in example_prompts]
with vllm_runner(
model, max_model_len=512, dtype=dtype, enable_prefix_caching=True
) as vllm_model:
vllm_config = vllm_model.llm.llm_engine.vllm_config
cache_config = vllm_config.cache_config
assert cache_config.enable_prefix_caching
# First Run
vllm_model.classify(example_prompts)
# assert prefix_caching works
pooling_outputs = vllm_model.llm.encode(
example_prompts, pooling_task="classify"
)
for output in pooling_outputs:
assert output.num_cached_tokens > 0
vllm_outputs = [req_output.outputs.data for req_output in pooling_outputs]
with hf_runner(
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
) as hf_model:
hf_outputs = hf_model.classify(example_prompts)
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = torch.tensor(hf_output)
vllm_output = torch.tensor(vllm_output)
assert torch.allclose(
hf_output, vllm_output, 1e-3 if dtype == "float" else 1e-2
)
@pytest.mark.parametrize(
"model",
["Qwen/Qwen3-Embedding-0.6B"],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_embed_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
):
# example_prompts is too short for testing prefix_caching
example_prompts = [str(s).strip() * 10 for s in example_prompts]
with vllm_runner(
model,
runner="pooling",
max_model_len=None,
enable_prefix_caching=True,
) as vllm_model:
vllm_config = vllm_model.llm.llm_engine.vllm_config
cache_config = vllm_config.cache_config
assert cache_config.enable_prefix_caching
# First Run
vllm_model.embed(example_prompts)
# assert prefix_caching works
pooling_outputs = vllm_model.llm.encode(example_prompts, pooling_task="embed")
for output in pooling_outputs:
assert output.num_cached_tokens > 0
vllm_outputs = [req_output.outputs.data for req_output in pooling_outputs]
with hf_runner(
model,
is_sentence_transformer=True,
) as hf_model:
run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
@pytest.mark.parametrize(
"model",
[
"intfloat/e5-small",
"Alibaba-NLP/gte-Qwen2-1.5B-instruct", # is_causal == False
"papluca/xlm-roberta-base-language-detection",
],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_non_causal_models(
hf_runner, vllm_runner, example_prompts, model: str, dtype: str
) -> None:
with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
vllm_config = vllm_model.llm.llm_engine.vllm_config
cache_config = vllm_config.cache_config
assert not cache_config.enable_prefix_caching
@@ -1,212 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import httpx
import openai
import pytest
import torch
from ....utils import RemoteOpenAIServer
from .embed_utils import run_client_embeddings
MODEL_NAME = "BAAI/bge-m3"
MAX_MODEL_LEN = 512
# Example from https://huggingface.co/BAAI/bge-m3
sentences_1 = ["What is BGE M3?", "Definition of BM25"]
sentences_2 = [
"BGE M3 is an embedding model supporting dense retrieval, "
"lexical matching and multi-vector interaction.",
"BM25 is a bag-of-words retrieval function that ranks a set "
"of documents based on the query terms appearing in each document",
]
similarity_reference = [[0.6259, 0.3474], [0.3309, 0.6734]]
lexical_score_reference = [0.19554901123046875, 0.0]
colbert_score_reference = [0.7797, 0.4620]
SUPPORTED_TASKS = ["embed", "token_embed", "token_classify"]
@pytest.fixture(scope="module", params=SUPPORTED_TASKS)
def pooling_task(request):
yield request.param
@pytest.fixture(scope="module")
def server(pooling_task):
args = [
"--max-model-len",
str(MAX_MODEL_LEN),
"--hf-overrides",
'{"architectures": ["BgeM3EmbeddingModel"]}',
"--pooler-config.task",
pooling_task,
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.mark.asyncio
async def test_bge_m3_api_server_embedding(server, pooling_task):
client = server.get_async_client()
if pooling_task != "embed":
with pytest.raises(openai.InternalServerError):
await run_client_embeddings(
client,
MODEL_NAME,
sentences_1,
)
return
embeddings_list_1 = await run_client_embeddings(
client,
MODEL_NAME,
sentences_1,
)
embeddings_list_2 = await run_client_embeddings(
client,
MODEL_NAME,
sentences_2,
)
embeddings_1 = torch.tensor(embeddings_list_1)
embeddings_2 = torch.tensor(embeddings_list_2)
similarity = embeddings_1 @ embeddings_2.T
# reference values from BAAI/bge-m3 documentation
reference = torch.tensor(similarity_reference)
assert torch.allclose(similarity, reference, rtol=0.01)
async def tokenize(client: openai.AsyncOpenAI, sentences: list[str]) -> list[list[int]]:
futures = []
for sentence in sentences:
futures.append(
client.post(
"../tokenize",
body={"model": MODEL_NAME, "prompt": sentence},
cast_to=httpx.Response,
)
)
return [(await future).json()["tokens"] for future in futures]
async def sparse_embeddings(
client: openai.AsyncOpenAI, sentences: list[str]
) -> list[dict[int, float]]:
all_tokens = await tokenize(client, sentences)
result = await client.post(
"../pooling",
body={"model": MODEL_NAME, "input": sentences, "task": "token_classify"},
cast_to=httpx.Response,
)
all_embeddings = [data["data"] for data in result.json()["data"]]
ret = []
for sent_tokens, sent_emb in zip(all_tokens, all_embeddings):
token_embs = dict[int, float]()
if sent_tokens[0] == 0:
sent_tokens = sent_tokens[1:]
for token, val in zip(sent_tokens, sent_emb):
token_embs[token] = max(val, token_embs.get(token, 0.0))
ret.append(token_embs)
return ret
# Based on https://github.com/FlagOpen/FlagEmbedding/blob/6fd176266f2382878bcc69cd656cff425d52f49b/FlagEmbedding/inference/embedder/encoder_only/m3.py#L129
def compute_lexical_matching_score(
lw1: dict[int, float], lw2: dict[int, float]
) -> float:
scores = 0.0
for token, weight in lw1.items():
if token in lw2:
scores += weight * lw2[token]
return scores
@pytest.mark.asyncio
async def test_bge_m3_api_server_sparse_embedding(server, pooling_task):
client = server.get_async_client()
if pooling_task != "token_classify":
with pytest.raises(openai.BadRequestError):
await sparse_embeddings(client, sentences_1)
return
embeddings_1 = await sparse_embeddings(client, sentences_1)
embeddings_2 = await sparse_embeddings(client, sentences_2)
lexical_scores_1_0_x_2_0 = compute_lexical_matching_score(
embeddings_1[0], embeddings_2[0]
)
assert lexical_scores_1_0_x_2_0 == pytest.approx(
lexical_score_reference[0], rel=0.01
)
lexical_scores_1_0_x_1_1 = compute_lexical_matching_score(
embeddings_1[0], embeddings_1[1]
)
assert lexical_scores_1_0_x_1_1 == pytest.approx(
lexical_score_reference[1], rel=0.01
)
@pytest.mark.asyncio
async def test_bge_m3_api_server_sparse_embedding_corner_case(server, pooling_task):
if pooling_task != "token_classify":
return
client = server.get_async_client()
embeddings = await sparse_embeddings(client, ["Hi"])
assert len(embeddings) == 1
assert 2673 in embeddings[0]
assert embeddings[0][2673] == pytest.approx(0.26710861921310425, rel=0.01)
# https://github.com/FlagOpen/FlagEmbedding/blob/6fd176266f2382878bcc69cd656cff425d52f49b/FlagEmbedding/inference/embedder/encoder_only/m3.py#L163
def colbert_score(q_reps: torch.Tensor, p_reps: torch.Tensor) -> torch.Tensor:
token_scores = torch.einsum("in,jn->ij", q_reps, p_reps)
scores, _ = token_scores.max(-1)
scores = torch.sum(scores) / q_reps.size(0)
return scores
@pytest.mark.asyncio
async def test_bge_m3_api_server_multi_vector(server, pooling_task):
client = server.get_async_client()
if pooling_task != "token_embed":
with pytest.raises(openai.BadRequestError):
await client.post(
"../pooling",
body={"model": MODEL_NAME, "input": sentences_1, "task": "token_embed"},
cast_to=httpx.Response,
)
return
result_1 = await client.post(
"../pooling",
body={"model": MODEL_NAME, "input": sentences_1, "task": "token_embed"},
cast_to=httpx.Response,
)
embeddings_1 = [torch.tensor(data["data"]) for data in result_1.json()["data"]]
result_2 = await client.post(
"../pooling",
body={"model": MODEL_NAME, "input": sentences_2, "task": "token_embed"},
cast_to=httpx.Response,
)
embeddings_2 = [torch.tensor(data["data"]) for data in result_2.json()["data"]]
colbert_score_1_0_x_2_0 = colbert_score(embeddings_1[0], embeddings_2[0])
assert colbert_score_1_0_x_2_0 == pytest.approx(
colbert_score_reference[0], rel=0.01
)
colbert_score_1_0_x_2_1 = colbert_score(embeddings_1[0], embeddings_2[1])
assert colbert_score_1_0_x_2_1 == pytest.approx(
colbert_score_reference[1], rel=0.01
)
@@ -1,108 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModelForSequenceClassification
from vllm.platforms import current_platform
@pytest.mark.parametrize(
"model",
[
pytest.param(
"jason9693/Qwen2.5-1.5B-apeach",
marks=[
pytest.mark.core_model,
pytest.mark.cpu_model,
pytest.mark.slow_test,
],
),
],
)
@pytest.mark.parametrize("dtype", ["half"] if current_platform.is_rocm() else ["float"])
def test_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.classify(example_prompts)
with hf_runner(
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
) as hf_model:
hf_outputs = hf_model.classify(example_prompts)
# check logits difference
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = torch.tensor(hf_output)
vllm_output = torch.tensor(vllm_output)
# the tolerance value of 1e-2 is selected based on the
# half datatype tests in
# tests/models/language/pooling/test_embedding.py
assert torch.allclose(
hf_output,
vllm_output,
rtol=2e-3 if dtype == "float" else 1e-2,
)
@pytest.mark.core_model
def test_bert_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
model = "cross-encoder/ms-marco-TinyBERT-L-2-v2"
score_inputs = (
"What is the capital of France?",
[
"Paris.",
"Paris is the capital and largest city of France.",
"William Shakespeare wrote Hamlet in the early seventeenth century.",
],
)
prompt_batches = [
["short input"],
[
"short input",
"a longer input that exercises mixed sequence lengths",
],
]
with hf_runner(
model, dtype="half", auto_cls=AutoModelForSequenceClassification
) as hf_model:
# HfRunner uses problem_type to preserve the model's
# sbert_ce_default_activation_function=Identity raw logits.
hf_model.config.problem_type = "regression"
hf_outputs = [hf_model.classify(prompts) for prompts in prompt_batches]
text_1, text_2 = score_inputs
text_pairs = [[text_1, document] for document in text_2]
with hf_runner(model, dtype="half", is_cross_encoder=True) as hf_model:
hf_scores = hf_model.predict(text_pairs).tolist()
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
runner="pooling",
dtype="half",
max_model_len=64,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = [vllm_model.classify(prompts) for prompts in prompt_batches]
vllm_scores = vllm_model.score(*score_inputs)
for hf_batch, vllm_batch in zip(hf_outputs, vllm_outputs):
hf_tensor = torch.tensor(hf_batch)
vllm_tensor = torch.tensor(vllm_batch)
assert vllm_tensor.shape == hf_tensor.shape
assert torch.allclose(vllm_tensor, hf_tensor, rtol=1e-2, atol=1e-4)
assert torch.allclose(
torch.tensor(vllm_scores),
torch.tensor(hf_scores),
rtol=1e-2,
atol=1e-4,
)
@@ -1,374 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ColBERT late interaction scoring.
Tests are parametrized across multiple ColBERT backbones to ensure the
generic ColBERT support works with different encoder architectures.
"""
from contextlib import contextmanager
import pytest
import torch
from tests.utils import wait_for_memory_to_settle
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
# -----------------------------------------------------------------------
# Model definitions: (model_name, colbert_dim, extra vllm_runner kwargs)
# -----------------------------------------------------------------------
COLBERT_MODELS = {
"bert": {
"model": "answerdotai/answerai-colbert-small-v1",
"colbert_dim": 96,
"max_model_len": 512,
"extra_kwargs": {},
"hf_comparison": {
"weights_file": "model.safetensors",
"weights_key": "linear.weight",
"trust_remote_code": False,
"model_cls": "BertModel",
},
},
"modernbert": {
"model": "lightonai/GTE-ModernColBERT-v1",
"colbert_dim": 128,
"max_model_len": 299,
"extra_kwargs": {
"hf_overrides": {
"architectures": ["ColBERTModernBertModel"],
},
},
"hf_comparison": {
"weights_file": "1_Dense/model.safetensors",
"weights_key": "linear.weight",
"trust_remote_code": False,
"model_cls": "AutoModel",
},
},
"jina": {
"model": "jinaai/jina-colbert-v2",
"colbert_dim": 128,
"max_model_len": 8192,
"extra_kwargs": {
"hf_overrides": {
"architectures": ["ColBERTJinaRobertaModel"],
},
},
"hf_comparison": {
"weights_file": "model.safetensors",
"weights_key": "linear.weight",
"trust_remote_code": True,
"model_cls": "AutoModel",
},
},
"lfm2": {
"model": "LiquidAI/LFM2-ColBERT-350M",
"colbert_dim": 128,
"max_model_len": 511,
"extra_kwargs": {
"hf_overrides": {
"architectures": ["ColBERTLfm2Model"],
},
},
"hf_comparison": {
"weights_file": "1_Dense/model.safetensors",
"weights_key": "linear.weight",
"trust_remote_code": False,
"model_cls": "AutoModel",
},
},
}
TEXTS_1 = [
"What is the capital of France?",
"What is the capital of Germany?",
]
TEXTS_2 = [
"The capital of France is Paris.",
"The capital of Germany is Berlin.",
]
DTYPE = "half"
def _load_hf_model(model_name: str, hf_spec: dict, device: torch.device):
"""Load HF model on the given device with a compatible attention impl."""
from transformers import AutoModel, BertModel
cls = BertModel if hf_spec["model_cls"] == "BertModel" else AutoModel
trust = hf_spec.get("trust_remote_code", False)
# Flash / Triton kernels require GPU tensors; fall back to eager on CPU.
extra = {}
if device.type == "cpu":
extra["attn_implementation"] = "eager"
model = cls.from_pretrained(
model_name,
trust_remote_code=trust,
**extra,
).to(device)
model.eval()
# Transformers 5.0 weight materialization can clear non-persistent
# buffers (e.g. rotary inv_freq) that were registered with
# persistent=False. Re-compute them so the model produces valid output.
for mod in model.modules():
if hasattr(mod, "_compute_inv_freq") and hasattr(mod, "inv_freq"):
mod.inv_freq = mod._compute_inv_freq(device=device)
return model
def _load_projection_weight(model_name: str, hf_spec: dict, device: torch.device):
"""Download and return the ColBERT linear projection weight."""
from safetensors.torch import load_file
from vllm.transformers_utils.repo_utils import hf_api, with_retry
path = with_retry(
lambda: hf_api().hf_hub_download(model_name, filename=hf_spec["weights_file"]),
f"Error downloading {hf_spec['weights_file']} from {model_name}",
)
weights = load_file(path)
return weights[hf_spec["weights_key"]].to(device)
def _compute_hf_colbert_embeddings(model, tokenizer, linear_weight, texts, device):
"""Run HF model + projection and return L2-normalised token embeddings."""
import torch.nn.functional as F
embeddings = []
for text in texts:
inputs = tokenizer(text, return_tensors="pt").to(device)
with torch.no_grad():
hidden = model(**inputs).last_hidden_state.float()
projected = F.linear(hidden, linear_weight.float())
normalised = F.normalize(projected, p=2, dim=-1)
embeddings.append(normalised.squeeze(0).cpu())
return embeddings
@contextmanager
def _hf_colbert_model(model_name: str, hf_spec: dict, device: torch.device):
"""Load the HF backbone + ColBERT projection, freeing the GPU on exit.
These live outside any runner context, so without explicit cleanup ROCm
keeps the VRAM resident and the next backend parametrization (or test)
OOMs on startup.
"""
hf_model = _load_hf_model(model_name, hf_spec, device)
linear_weight = _load_projection_weight(model_name, hf_spec, device)
try:
yield hf_model, linear_weight
finally:
del hf_model, linear_weight
cleanup_dist_env_and_memory()
wait_for_memory_to_settle()
def _assert_embeddings_close(vllm_outputs, hf_embeddings):
"""Assert that vLLM and HuggingFace embeddings match."""
for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)):
vllm_emb = torch.as_tensor(vllm_out).float()
assert hf_emb.shape == vllm_emb.shape, (
f"Shape mismatch for text {i}: HF {hf_emb.shape} vs vLLM {vllm_emb.shape}"
)
torch.testing.assert_close(
vllm_emb,
hf_emb,
rtol=1e-2,
atol=1e-2,
msg=f"Embedding mismatch for text {i}",
)
@pytest.fixture(params=list(COLBERT_MODELS.keys()), scope="class")
def colbert_spec(request):
"""Return the model spec dict for the current parametrization."""
return COLBERT_MODELS[request.param]
@pytest.fixture(scope="class")
def colbert_model_name(colbert_spec):
return colbert_spec["model"]
@pytest.fixture(scope="class")
def colbert_dim(colbert_spec):
return colbert_spec["colbert_dim"]
@pytest.fixture(scope="class")
def colbert_max_model_len(colbert_spec):
return colbert_spec["max_model_len"]
@pytest.fixture(scope="class")
def colbert_extra_kwargs(colbert_spec):
return colbert_spec["extra_kwargs"]
@pytest.fixture(scope="class")
def colbert_model(
vllm_runner,
colbert_model_name,
colbert_max_model_len,
colbert_extra_kwargs,
):
with vllm_runner(
colbert_model_name,
runner="pooling",
dtype=DTYPE,
max_model_len=colbert_max_model_len,
enforce_eager=True,
**colbert_extra_kwargs,
) as vllm_model:
yield vllm_model
class TestColbertSharedEngine:
"""Tests sharing one engine per model.
Class-scoped so the engine is released before `test_colbert_hf_comparison`,
which needs a runner of its own and cannot start while this one holds VRAM.
"""
def test_colbert_token_embed(self, colbert_model, colbert_dim):
"""Test that ColBERT model produces token embeddings."""
outputs = colbert_model.token_embed([TEXTS_1[0]])
assert len(outputs) == 1
emb = torch.as_tensor(outputs[0])
assert emb.dim() == 2
assert emb.shape[1] == colbert_dim
assert emb.shape[0] > 1
def test_colbert_late_interaction_1_to_1(self, colbert_model):
"""Test ColBERT late interaction scoring with 1:1 query-document pair."""
q_outputs = colbert_model.token_embed([TEXTS_1[0]])
d_outputs = colbert_model.token_embed([TEXTS_2[0]])
q_emb = torch.as_tensor(q_outputs[0])
d_emb = torch.as_tensor(d_outputs[0])
manual_score = compute_maxsim_score(q_emb, d_emb).item()
vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2[0])
assert len(vllm_scores) == 1
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
def test_colbert_late_interaction_1_to_N(self, colbert_model):
"""Test ColBERT late interaction scoring with 1:N query-documents."""
q_outputs = colbert_model.token_embed([TEXTS_1[0]])
d_outputs = colbert_model.token_embed(TEXTS_2)
q_emb = torch.as_tensor(q_outputs[0])
manual_scores = []
for d_out in d_outputs:
d_emb = torch.as_tensor(d_out)
manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2)
assert len(vllm_scores) == 2
for i in range(2):
assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
def test_colbert_late_interaction_N_to_N(self, colbert_model):
"""Test ColBERT late interaction scoring with N:N query-documents."""
q_outputs = colbert_model.token_embed(TEXTS_1)
d_outputs = colbert_model.token_embed(TEXTS_2)
manual_scores = []
for q_out, d_out in zip(q_outputs, d_outputs):
q_emb = torch.as_tensor(q_out)
d_emb = torch.as_tensor(d_out)
manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
vllm_scores = colbert_model.score(TEXTS_1, TEXTS_2)
assert len(vllm_scores) == 2
for i in range(2):
assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
def test_colbert_relevance_ordering(self, colbert_model):
"""Test that ColBERT scores relevant documents higher than irrelevant."""
query = "What is machine learning?"
documents = [
"Machine learning is a subset of artificial intelligence.",
"Python is a programming language.",
"Deep learning uses neural networks.",
]
scores = colbert_model.score(query, documents)
assert len(scores) == 3
assert scores[0] > scores[1], "ML doc should score higher than Python doc"
assert scores[2] > scores[1], "DL doc should score higher than Python doc"
def test_colbert_embed_not_supported(self, colbert_model):
"""Test that ColBERT model does not support the embed task."""
with pytest.raises(ValueError, match="Embedding API is not supported"):
colbert_model.embed([TEXTS_1[0]])
@pytest.mark.parametrize(
("backend", "use_v2"),
[
pytest.param("bert", True, id="bert-v2"),
pytest.param("modernbert", True, id="modernbert-v2"),
pytest.param("jina", True, id="jina-v2"),
pytest.param("lfm2", True, id="lfm2-v2"),
],
)
def test_colbert_hf_comparison(vllm_runner, monkeypatch, backend, use_v2):
"""Test that vLLM ColBERT embeddings match HuggingFace for each backend."""
from transformers import AutoTokenizer
spec = COLBERT_MODELS[backend]
hf_spec = spec["hf_comparison"]
extra_kwargs = spec["extra_kwargs"]
model_name = spec["model"]
assert isinstance(model_name, str)
assert isinstance(hf_spec, dict)
assert isinstance(extra_kwargs, dict)
test_texts = [TEXTS_1[0], TEXTS_2[0]]
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2 else "0")
with vllm_runner(
model_name,
runner="pooling",
dtype="float32",
max_model_len=spec["max_model_len"],
enforce_eager=True,
**extra_kwargs,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner == use_v2
vllm_outputs = vllm_model.token_embed(test_texts)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
hf_tokenizer = AutoTokenizer.from_pretrained(
model_name,
trust_remote_code=hf_spec.get("trust_remote_code", False),
)
with _hf_colbert_model(model_name, hf_spec, device) as (hf_model, linear_weight):
hf_embeddings = _compute_hf_colbert_embeddings(
hf_model,
hf_tokenizer,
linear_weight,
test_texts,
device,
)
_assert_embeddings_close(vllm_outputs, hf_embeddings)
@@ -1,219 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModel
from vllm import PoolingParams
from vllm.config import PoolerConfig
from ...utils import check_embeddings_close
@pytest.mark.parametrize(
"model",
[
# Be careful of the order of models, decoder-only models should be
# placed before encoder-only models, otherwise `Qwen2.5-0.5B-Instruct`
# case won't pass because gte-Qwen2-1.5B-instruct will cache custom
# model code with bidirectional attention.
# [Decoder-only]
pytest.param(
"BAAI/bge-multilingual-gemma2",
marks=[pytest.mark.core_model, pytest.mark.slow_test],
),
pytest.param(
"intfloat/e5-mistral-7b-instruct",
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
),
pytest.param(
"ssmits/Qwen2-7B-Instruct-embed-base", marks=[pytest.mark.cpu_model]
),
# [Encoder-only]
pytest.param(
"BAAI/bge-base-en-v1.5",
marks=[
pytest.mark.core_model,
pytest.mark.cpu_model,
pytest.mark.slow_test,
],
),
pytest.param("sentence-transformers/all-MiniLM-L12-v2"),
pytest.param("intfloat/multilingual-e5-small"),
# [Cross-Encoder]
pytest.param(
"sentence-transformers/stsb-roberta-base-v2",
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
),
],
)
def test_models(
hf_runner,
vllm_runner,
example_prompts,
model,
) -> None:
vllm_extra_kwargs = {}
if model == "ssmits/Qwen2-7B-Instruct-embed-base":
vllm_extra_kwargs["pooler_config"] = PoolerConfig(
seq_pooling_type="MEAN", use_activation=False
)
max_model_len: int | None = 512
if model in [
"sentence-transformers/all-MiniLM-L12-v2",
"sentence-transformers/stsb-roberta-base-v2",
]:
max_model_len = None
# The example_prompts has ending "\n", for example:
# "Write a short story about a robot that dreams for the first time.\n"
# sentence_transformers will strip the input texts, see:
# https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
# This makes the input_ids different between hf_model and vllm_model.
# So we need to strip the input texts to avoid test failing.
example_prompts = [str(s).strip() for s in example_prompts]
with hf_runner(model, is_sentence_transformer=True) as hf_model:
hf_outputs = hf_model.encode(example_prompts)
with vllm_runner(
model, runner="pooling", max_model_len=max_model_len, **vllm_extra_kwargs
) as vllm_model:
vllm_outputs = vllm_model.embed(example_prompts)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
@pytest.mark.parametrize(
"model",
[
"BAAI/bge-base-en-v1.5",
"intfloat/multilingual-e5-small",
],
)
@torch.inference_mode()
def test_encoder_only_model_runner_v2_attention(
hf_runner,
vllm_runner,
monkeypatch,
model: str,
) -> None:
prompts = [
"short input",
"a longer input that exercises mixed sequence lengths",
]
with hf_runner(model, dtype="float", auto_cls=AutoModel) as hf_model:
hf_outputs = []
for prompt in prompts:
inputs = hf_model.tokenizer(prompt, return_tensors="pt")
output = hf_model.model(**hf_model.wrap_device(inputs))
embedding = torch.nn.functional.normalize(
output.last_hidden_state[0, -1].float(), dim=0
)
hf_outputs.append(embedding.cpu().tolist())
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
runner="pooling",
dtype="float",
max_model_len=64,
max_num_seqs=2,
gpu_memory_utilization=0.25,
pooler_config=PoolerConfig(
task="embed", seq_pooling_type="LAST", use_activation=True
),
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = vllm_model.embed(prompts)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
@pytest.mark.core_model
def test_encoder_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
model = "sentence-transformers/all-MiniLM-L6-v2"
prompt_batches = [
["short input"],
[
"short input",
"a longer input that exercises mixed sequence lengths",
],
]
with hf_runner(model, is_sentence_transformer=True) as hf_model:
hf_outputs = [hf_model.encode(prompts) for prompts in prompt_batches]
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
runner="pooling",
max_model_len=64,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = [vllm_model.embed(prompts) for prompts in prompt_batches]
for hf_batch, vllm_batch in zip(hf_outputs, vllm_outputs):
check_embeddings_close(
embeddings_0_lst=hf_batch,
embeddings_1_lst=vllm_batch,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
@pytest.mark.core_model
def test_matryoshka_dimensions_model_runner_v2(
hf_runner, vllm_runner, monkeypatch
) -> None:
model = "Snowflake/snowflake-arctic-embed-m-v1.5"
prompts = ["short input", "a longer input for a different output width"]
dimensions = [None, 256]
with hf_runner(model, is_sentence_transformer=True) as hf_model:
hf_outputs = hf_model.encode(prompts)
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
runner="pooling",
max_model_len=64,
gpu_memory_utilization=0.25,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = vllm_model.embed(
prompts,
pooling_params=[PoolingParams(dimensions=d) for d in dimensions],
)
expected_outputs = []
for output, dimension in zip(hf_outputs, dimensions):
output = torch.as_tensor(output)
if dimension is not None:
output = torch.nn.functional.normalize(output[:dimension], dim=0)
expected_outputs.append(output.tolist())
assert [len(output) for output in vllm_outputs] == [768, 256]
check_embeddings_close(
embeddings_0_lst=expected_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
@@ -1,48 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm import TokensPrompt
from vllm.config import PoolerConfig
@pytest.mark.parametrize(
"model",
["Qwen/Qwen3-0.6B"],
)
@torch.inference_mode
def test_extract_hidden_states(hf_runner, vllm_runner, model: str):
n_prompt_tokens = [55, 56, 57]
token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
with vllm_runner(
model,
max_model_len=128,
enforce_eager=True,
runner="pooling",
pooler_config=PoolerConfig(task="token_embed"),
enable_prefix_caching=True,
) as vllm_model:
pooling_outputs = vllm_model.llm.encode(
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
pooling_task="token_embed",
)
for n, output in zip(n_prompt_tokens, pooling_outputs):
assert len(output.prompt_token_ids) == n
assert len(output.outputs.data) == n
assert output.num_cached_tokens == 0
# test enable_prefix_caching plus all pooling
# we need to skip reading cache at this request by
# request.skip_reading_prefix_cache
pooling_outputs = vllm_model.llm.encode(
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
pooling_task="token_embed",
)
for n, output in zip(n_prompt_tokens, pooling_outputs):
assert len(output.prompt_token_ids) == n
assert len(output.outputs.data) == n
assert output.num_cached_tokens == 0
@@ -1,47 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModelForSequenceClassification
@pytest.mark.parametrize(
"model",
["nie3e/sentiment-polish-gpt2-small"],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_classify_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with hf_runner(
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
) as hf_model:
hf_outputs = hf_model.classify(example_prompts)
for head_dtype_str in ["float32", "model"]:
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
hf_overrides={"head_dtype": head_dtype_str},
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
model_dtype = model_config.dtype
head_dtype = model_config.head_dtype
if head_dtype_str == "float32":
assert head_dtype == torch.float32
elif head_dtype_str == "model":
assert head_dtype == model_dtype
vllm_outputs = vllm_model.classify(example_prompts)
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = torch.tensor(hf_output).float()
vllm_output = torch.tensor(vllm_output).float()
assert torch.allclose(hf_output, vllm_output, atol=1e-2)
@@ -1,67 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Backbone validation for Jina Embeddings V5.
The V5 family ships two backbones under one `architectures` entry: `-small` is
a Qwen3 decoder, while `-nano` is a bidirectional EuroBERT encoder. Upstream
ships a separate `configuration_*.py` per repository, so the only signal
distinguishing them is `is_decoder`, which the encoder variant sets to False.
`JinaEmbeddingsV5ModelConfig` uses it to enable bidirectional attention for the
encoder variant; `JinaEmbeddingsV5Model` then dispatches to the correct backbone.
"""
from types import SimpleNamespace
from typing import cast
import pytest
from transformers import PreTrainedConfig
from vllm.config import ModelConfig
from vllm.model_executor.models.config import (
MODELS_CONFIG_MAP,
JinaEmbeddingsV5ModelConfig,
)
def _model_config(hf_config: PreTrainedConfig) -> ModelConfig:
"""Minimal stand-in for ModelConfig; only hf_config is read."""
return cast(ModelConfig, SimpleNamespace(hf_config=hf_config))
@pytest.mark.cpu_test
def test_registered_for_the_architecture():
"""The handler only runs if it is wired to the architecture name."""
assert MODELS_CONFIG_MAP["JinaEmbeddingsV5Model"] is JinaEmbeddingsV5ModelConfig
@pytest.mark.cpu_test
def test_encoder_backbone_enables_bidirectional_attention():
"""An encoder checkpoint (is_decoder=False) is supported.
The handler sets is_causal=False so the Llama backbone uses
EncoderOnlyAttention; JinaEmbeddingsV5Model then dispatches to the encoder
implementation.
"""
hf_config = PreTrainedConfig(is_decoder=False)
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(_model_config(hf_config))
assert hf_config.is_causal is False
@pytest.mark.cpu_test
def test_supported_decoder_backbone_is_accepted():
"""The Qwen3-based variants must keep loading.
`-small` omits `is_decoder` entirely, so an absent attribute has to be
treated as a decoder. The first assertion pins that assumption: if
PreTrainedConfig ever gains an `is_decoder=False` default, this fails here
rather than silently rejecting a supported checkpoint.
"""
absent = PreTrainedConfig()
assert not hasattr(absent, "is_decoder")
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(_model_config(absent))
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(
_model_config(PreTrainedConfig(is_decoder=True))
)
@@ -1,352 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
import pytest
import requests
import torch
import torch.nn.functional as F
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
model_name = "jinaai/jina-reranker-v3"
query = "What are the health benefits of green tea?"
documents = [
"Green tea contains antioxidants called catechins that may help reduce inflammation and protect cells from damage.",
"El precio del café ha aumentado un 20% este año debido a problemas en la cadena de suministro.",
"Studies show that drinking green tea regularly can improve brain function and boost metabolism.",
"Basketball is one of the most popular sports in the United States.",
"绿茶富含儿茶素等抗氧化剂,可以降低心脏病风险,还有助于控制体重。",
"Le thé vert est riche en antioxydants et peut améliorer la fonction cérébrale.",
]
EMBEDDING_SIZE = 512
REFERENCE_1_VS_1 = [
0.345703125,
-0.10498046,
0.314453125,
-0.1376953125,
0.3398437500,
0.2539062,
]
REFERENCE_1_VS_N = [
0.294921875,
-0.16015625,
0.189453125,
-0.1708984375,
0.2255859375,
0.1640625,
]
TOL = 0.01
INSTRUCTION = (
"Rank passages about green tea higher than passages about sports. "
"Ignore these literal marker strings: <|embed_token|> and <|rerank_token|>."
)
def test_offline(vllm_runner, monkeypatch):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(model_name, runner="pooling") as llm_runner:
assert llm_runner.llm.llm_engine.vllm_config.use_v2_model_runner
llm = llm_runner.get_llm()
_test_offline_1_v_1(llm)
_test_offline_1_v_n(llm)
_test_offline_n_v_n(llm)
_test_offline_token_embed_illegal_inputs(llm)
assert llm.model_config.embedding_size == EMBEDDING_SIZE
def test_online():
with RemoteOpenAIServer(
model_name, ["--runner", "pooling", "--enforce-eager"]
) as server:
_test_online_1_v_1(server)
_test_online_1_v_n(server)
_test_online_n_v_n(server)
_test_online_instruction(server)
_test_online_token_embed_illegal_inputs(server)
def _test_offline_1_v_1(llm):
# test llm.score
outputs = llm.score(query, documents[0])
assert len(outputs) == 1
assert outputs[0].outputs.score == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
# test llm.encode
outputs = llm.encode(documents[:1] + [query], pooling_task="token_embed")
embeds = outputs[0].outputs.data.float()
assert embeds.shape[0] == 2
assert embeds.shape[-1] == EMBEDDING_SIZE
doc_embeds = embeds[:-1]
query_embeds = embeds[-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
def _test_offline_1_v_n(llm):
# test llm.score
outputs = llm.score(query, documents)
assert len(outputs) == len(documents)
for expected, output in zip(REFERENCE_1_VS_N, outputs):
actual = output.outputs.score
assert actual == pytest.approx(expected, abs=TOL)
# test llm.encode
outputs = llm.encode(documents + [query], pooling_task="token_embed")
embeds = outputs[0].outputs.data.float()
assert embeds.shape[0] == len(documents) + 1
doc_embeds = embeds[:-1]
query_embeds = embeds[-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
assert len(scores) == len(documents)
for expected, actual in zip(REFERENCE_1_VS_N, scores):
assert actual == pytest.approx(expected, abs=TOL)
def _test_offline_n_v_n(llm):
# test llm.score
outputs = llm.score([query] * len(documents), documents)
assert len(outputs) == len(documents)
for expected, output in zip(REFERENCE_1_VS_1, outputs):
actual = output.outputs.score
assert actual == pytest.approx(expected, abs=TOL)
# test llm.encode
for doc, expected in zip(documents, REFERENCE_1_VS_1):
outputs = llm.encode([doc, query], pooling_task="token_embed")
embeds = outputs[0].outputs.data.float()
assert embeds.shape[0] == 2
doc_embeds = embeds[:-1]
query_embeds = embeds[-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
assert scores[0] == pytest.approx(expected, abs=TOL)
def _test_offline_token_embed_illegal_inputs(llm):
with pytest.raises(
ValueError, match="The JinaForRanking model requires at least 2 inputs."
):
llm.encode([query], pooling_task="token_embed")
with pytest.raises(
ValueError, match="The JinaForRanking model only supports text as input."
):
llm.encode([1, 2, 3], pooling_task="token_embed")
def _get_score_response(server, query, document, **extra_body):
payload = {
"model": model_name,
"queries": query,
"documents": document,
}
payload.update(extra_body)
score_response = requests.post(
server.url_for("score"),
json=payload,
)
score_response.raise_for_status()
return ScoreResponse.model_validate(score_response.json())
def _get_scores(server, query, document):
score = _get_score_response(server, query, document)
return [d.score for d in score.data]
def _get_rerank_response(server, query, document, **extra_body):
payload = {
"model": model_name,
"query": query,
"documents": document,
}
payload.update(extra_body)
rerank_response = requests.post(
server.url_for("rerank"),
json=payload,
)
rerank_response.raise_for_status()
return RerankResponse.model_validate(rerank_response.json())
def _get_embeds(server, prompts: list[str]):
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"task": "token_embed",
"input": prompts,
"encoding_format": "float",
},
)
response.raise_for_status()
poolings = PoolingResponse.model_validate(response.json())
return torch.as_tensor([d.data for d in poolings.data][0]).float()
def _test_online_1_v_1(server):
# test scoring api
scores = _get_scores(server, query, documents[0])
assert len(scores) == 1
assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
# test pooling api
embeds = _get_embeds(server, [documents[0], query])
assert embeds.shape[0] == 2
assert embeds.shape[-1] == EMBEDDING_SIZE
doc_embeds = embeds[:-1]
query_embeds = embeds[-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
def _test_online_1_v_n(server):
# test scoring api
scores = _get_scores(server, query, documents)
assert len(scores) == len(documents)
for expected, actual in zip(REFERENCE_1_VS_N, scores):
assert actual == pytest.approx(expected, abs=TOL)
# test pooling api
embeds = _get_embeds(server, documents + [query])
assert embeds.shape[0] == len(documents) + 1
doc_embeds = embeds[:-1]
query_embeds = embeds[-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
assert len(scores) == len(documents)
for expected, actual in zip(REFERENCE_1_VS_N, scores):
assert actual == pytest.approx(expected, abs=TOL)
def _test_online_n_v_n(server):
# test scoring api
scores = _get_scores(server, [query] * len(documents), documents)
assert len(scores) == len(documents)
for expected, actual in zip(REFERENCE_1_VS_1, scores):
assert actual == pytest.approx(expected, abs=TOL)
# test pooling api
for doc, expected in zip(documents, REFERENCE_1_VS_1):
embeds = _get_embeds(server, [doc, query])
assert embeds.shape[0] == 2
doc_embeds = embeds[:-1]
query_embeds = embeds[-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
assert len(scores) == 1
assert scores[0] == pytest.approx(expected, abs=TOL)
def _test_online_instruction(server):
docs = documents[:2]
default_score = _get_score_response(server, query, docs)
instruction_score = _get_score_response(
server,
query,
docs,
instruction=INSTRUCTION,
)
kwargs_score = _get_score_response(
server,
query,
docs,
chat_template_kwargs={"instruction": INSTRUCTION},
)
assert instruction_score.usage.prompt_tokens > default_score.usage.prompt_tokens
assert kwargs_score.usage.prompt_tokens == instruction_score.usage.prompt_tokens
assert len(instruction_score.data) == len(default_score.data)
assert [d.score for d in kwargs_score.data] == pytest.approx(
[d.score for d in instruction_score.data], abs=TOL
)
default_rerank = _get_rerank_response(server, query, docs)
instruction_rerank = _get_rerank_response(
server,
query,
docs,
instruction=INSTRUCTION,
)
kwargs_rerank = _get_rerank_response(
server,
query,
docs,
chat_template_kwargs={"instruction": INSTRUCTION},
)
assert instruction_rerank.usage.prompt_tokens > default_rerank.usage.prompt_tokens
assert kwargs_rerank.usage.prompt_tokens == instruction_rerank.usage.prompt_tokens
assert len(instruction_rerank.results) == len(default_rerank.results)
assert [r.relevance_score for r in kwargs_rerank.results] == pytest.approx(
[r.relevance_score for r in instruction_rerank.results], abs=TOL
)
def _test_online_token_embed_illegal_inputs(server):
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"task": "token_embed",
"input": [query],
"encoding_format": "float",
},
)
assert response.json()["error"]["message"].startswith(
"The JinaForRanking model requires at least 2 inputs."
)
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"task": "token_embed",
"input": [1, 2, 3],
"encoding_format": "float",
},
)
assert response.json()["error"]["message"].startswith(
"The JinaForRanking model only supports text as input."
)
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"task": "token_embed",
"messages": [
{
"role": "user",
"content": "The cat sat on the mat.",
}
],
"encoding_format": "float",
},
)
assert response.json()["error"]["message"].startswith(
"The JinaForRanking does not support chat Request."
)
@@ -1,227 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for max_tokens_per_doc and max_tokens_per_query."""
import json
import os
from dataclasses import dataclass
import pytest
import requests
from tests.utils import VLLM_PATH, RemoteOpenAIServer
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse
TEMPLATE_DIR = str(VLLM_PATH / "examples/pooling/score/template")
ExpectedPromptTokens = int | tuple[int, ...]
long_query = "What is the capital of France?" * 20
long_doc = "The capital of France is Paris. " * 20
@dataclass
class TestConfig:
model: str
args: list[str]
without_truncated_prompt_tokens: ExpectedPromptTokens
with_max_tokens_per_query_prompt_tokens: ExpectedPromptTokens
with_max_tokens_per_doc_prompt_tokens: ExpectedPromptTokens
with_max_tokens_per_query_and_doc_prompt_tokens: ExpectedPromptTokens
RERANK_CONFIGS = [
# 1. cross-encoder
TestConfig(
model="jinaai/jina-reranker-v2-base-multilingual",
args=[
"--enforce-eager",
"--max-model-len",
"1024",
"--trust-remote-code",
],
without_truncated_prompt_tokens=284,
with_max_tokens_per_query_prompt_tokens=154,
with_max_tokens_per_doc_prompt_tokens=154,
with_max_tokens_per_query_and_doc_prompt_tokens=24,
),
# 2. cross-encoder + score template
TestConfig(
model="Qwen/Qwen3-Reranker-0.6B",
args=[
"--enforce-eager",
"--max-model-len",
"1024",
"--hf-overrides",
json.dumps(
{
"architectures": ["Qwen3ForSequenceClassification"],
"classifier_from_token": ["no", "yes"],
"is_original_qwen3_reranker": True,
}
),
"--chat-template",
os.path.join(TEMPLATE_DIR, "qwen3_reranker.jinja"),
],
without_truncated_prompt_tokens=352,
with_max_tokens_per_query_prompt_tokens=223,
with_max_tokens_per_doc_prompt_tokens=221,
with_max_tokens_per_query_and_doc_prompt_tokens=92,
),
# 3. bi-encoder
TestConfig(
model="intfloat/multilingual-e5-small",
args=[
"--enforce-eager",
"--max-model-len",
"512",
"--trust-remote-code",
],
# This model has produced both prompt-token totals in CI/local cache;
# keep truncation checks exact while tolerating the boundary delta.
without_truncated_prompt_tokens=(284, 285),
with_max_tokens_per_query_prompt_tokens=(154, 155),
with_max_tokens_per_doc_prompt_tokens=154,
with_max_tokens_per_query_and_doc_prompt_tokens=24,
),
# 4. late-interaction
TestConfig(
model="answerdotai/answerai-colbert-small-v1",
args=[
"--enforce-eager",
"--max-model-len",
"512",
"--trust-remote-code",
],
without_truncated_prompt_tokens=284,
with_max_tokens_per_query_prompt_tokens=154,
with_max_tokens_per_doc_prompt_tokens=154,
with_max_tokens_per_query_and_doc_prompt_tokens=24,
),
# 5. jinaai/jina-reranker-v3
TestConfig(
model="jinaai/jina-reranker-v3",
args=[
"--enforce-eager",
"--max-model-len",
"1024",
"--trust-remote-code",
],
without_truncated_prompt_tokens=567,
with_max_tokens_per_query_prompt_tokens=308,
with_max_tokens_per_doc_prompt_tokens=436,
with_max_tokens_per_query_and_doc_prompt_tokens=177,
),
]
def assert_prompt_tokens(actual: int, expected: ExpectedPromptTokens) -> None:
if isinstance(expected, int):
assert actual == expected
else:
assert actual in expected
@pytest.fixture(scope="module", params=RERANK_CONFIGS, ids=lambda c: c.model)
def server(request):
config: TestConfig = request.param
env_dict = {"VLLM_LOGGING_LEVEL": "WARNING"}
with RemoteOpenAIServer(
config.model, config.args, env_dict=env_dict
) as remote_server:
yield config, remote_server
def test_without_truncated(server):
"""Test that max_tokens_per_doc truncates documents correctly."""
config, remote_server = server
response = requests.post(
remote_server.url_for("rerank"),
json={"model": config.model, "query": long_query, "documents": [long_doc]},
)
response.raise_for_status()
rerank = RerankResponse.model_validate(response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 1
assert_prompt_tokens(
rerank.usage.prompt_tokens,
config.without_truncated_prompt_tokens,
)
def test_max_tokens_per_query(server):
"""Test that max_tokens_per_doc truncates documents correctly."""
config, remote_server = server
response = requests.post(
remote_server.url_for("rerank"),
json={
"model": config.model,
"query": long_query,
"documents": [long_doc],
"max_tokens_per_query": 10,
},
)
response.raise_for_status()
rerank = RerankResponse.model_validate(response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 1
assert_prompt_tokens(
rerank.usage.prompt_tokens,
config.with_max_tokens_per_query_prompt_tokens,
)
def test_max_tokens_per_doc(server):
"""Test that max_tokens_per_doc truncates documents correctly."""
config, remote_server = server
response = requests.post(
remote_server.url_for("rerank"),
json={
"model": config.model,
"query": long_query,
"documents": [long_doc],
"max_tokens_per_doc": 10,
},
)
response.raise_for_status()
rerank = RerankResponse.model_validate(response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 1
assert_prompt_tokens(
rerank.usage.prompt_tokens,
config.with_max_tokens_per_doc_prompt_tokens,
)
def test_max_tokens_per_query_and_doc(server):
"""Test that max_tokens_per_doc truncates documents correctly."""
config, remote_server = server
response = requests.post(
remote_server.url_for("rerank"),
json={
"model": config.model,
"query": long_query,
"documents": [long_doc],
"max_tokens_per_query": 10,
"max_tokens_per_doc": 10,
},
)
response.raise_for_status()
rerank = RerankResponse.model_validate(response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 1
assert_prompt_tokens(
rerank.usage.prompt_tokens,
config.with_max_tokens_per_query_and_doc_prompt_tokens,
)
@@ -1,104 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.config.pooler import PoolerConfig
def test_idefics_multimodal(
vllm_runner,
) -> None:
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
with vllm_runner(
model_name="HuggingFaceM4/Idefics3-8B-Llama3",
runner="pooling",
convert="classify",
load_format="dummy",
max_model_len=512,
enforce_eager=True,
tensor_parallel_size=1,
disable_log_stats=True,
dtype="bfloat16",
) as vllm_model:
llm = vllm_model.get_llm()
outputs = llm.classify(prompts)
for output in outputs:
assert len(output.outputs.probs) == 2
def update_config(config):
text_config = config.get_text_config()
text_config.update(
{
"architectures": ["Gemma3ForSequenceClassification"],
"classifier_from_token": ["A", "B", "C", "D", "E"],
"method": "no_post_processing",
"id2label": {
"A": "Chair",
"B": "Couch",
"C": "Table",
"D": "Bed",
"E": "Cupboard",
},
}
)
return config
def test_gemma_multimodal(
vllm_runner,
) -> None:
messages = [
{
"role": "system",
"content": """
You are a helpful assistant. You will be given a product description
which may also include an image. Classify the following product into
one of the categories:
A = chair
B = couch
C = table
D = bed
E = cupboard
You'll answer with exactly one letter (A, B, C, D, or E).""",
},
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/red_chair.jpg"
},
},
{"type": "text", "text": "A fine 19th century piece of furniture."},
],
},
]
with vllm_runner(
model_name="google/gemma-3-4b-it",
runner="pooling",
convert="classify",
load_format="auto",
hf_overrides=update_config,
pooler_config=PoolerConfig(seq_pooling_type="LAST"),
max_model_len=512,
enforce_eager=True,
tensor_parallel_size=1,
disable_log_stats=True,
dtype="bfloat16",
) as vllm_model:
llm = vllm_model.get_llm()
prompts = llm._preprocess_chat([messages])
result = llm.classify(prompts)
assert result[0].outputs.probs[0] > 0.95
assert all(c < 0.05 for c in result[0].outputs.probs[1:])
@@ -1,44 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModel
from tests.models.utils import check_embeddings_close
from vllm.config import PoolerConfig
@pytest.mark.parametrize(
"model",
["BAAI/bge-m3"],
)
@pytest.mark.parametrize("dtype", ["float"])
@torch.inference_mode
def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype: str):
with vllm_runner(
model,
runner="pooling",
pooler_config=PoolerConfig(task="token_embed"),
max_model_len=None,
dtype=dtype,
) as vllm_model:
vllm_outputs = vllm_model.token_embed(example_prompts)
with hf_runner(model, auto_cls=AutoModel, dtype=dtype) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in example_prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
embedding = output.last_hidden_state[0].float()
hf_outputs.append(embedding.cpu())
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
check_embeddings_close(
embeddings_0_lst=hf_output,
embeddings_1_lst=vllm_output,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
@@ -1,34 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModelForSequenceClassification
@pytest.mark.parametrize(
"model",
["Rami/multi-label-class-classification-on-github-issues"],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_classify_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.classify(example_prompts)
with hf_runner(
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
) as hf_model:
hf_outputs = hf_model.classify(example_prompts)
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = torch.tensor(hf_output)
vllm_output = torch.tensor(vllm_output)
assert torch.allclose(
hf_output, vllm_output, 1e-3 if dtype == "float" else 1e-2
)
@@ -1,129 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: SIM117
from functools import partial
import pytest
from ...utils import EmbedModelInfo
MODELS = [
EmbedModelInfo(
"nomic-ai/nomic-embed-text-v1",
# Fixme:
# Update nomic-embed code to support the latest
# HF version and remove revision set.
revision="720244025c1a7e15661a174c63cce63c8218e52b",
),
# EmbedModelInfo("nomic-ai/nomic-embed-text-v1.5"),
# EmbedModelInfo("nomic-ai/CodeRankEmbed"),
EmbedModelInfo("nomic-ai/nomic-embed-text-v2-moe"),
# EmbedModelInfo("Snowflake/snowflake-arctic-embed-m-long"),
]
rope_theta = 1000
factor = 4.0
original_max_position_embeddings = 2048
max_model_len = int(original_max_position_embeddings * factor)
@pytest.fixture(scope="module", autouse=True)
def predownload_models():
"""Warm the HF cache so engine startup never hits transient hub errors."""
from vllm.transformers_utils.repo_utils import hf_api, with_retry
for model_info in MODELS:
with_retry(
partial(
hf_api().snapshot_download,
model_info.name,
revision=model_info.revision,
),
f"Error downloading {model_info.name}",
)
@pytest.mark.parametrize("model_info", MODELS)
def test_default(model_info, vllm_runner):
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=None,
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
if model_info.name == "nomic-ai/nomic-embed-text-v2-moe":
# For nomic-embed-text-v2-moe the length is set to 512
# by sentence_bert_config.json.
assert model_config.max_model_len == 512
if model_info.name == "nomic-ai/nomic-embed-text-v1":
assert model_config.max_model_len == 8192
@pytest.mark.parametrize("model_info", MODELS)
def test_set_max_model_len_legal(model_info, vllm_runner):
# set max_model_len <= 512
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=256,
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
assert model_config.max_model_len == 256
# For nomic-embed-text-v2-moe the length is set to 512
# by sentence_bert_config.json.
if model_info.name == "nomic-ai/nomic-embed-text-v2-moe":
with pytest.raises(ValueError):
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=1024,
):
pass
return
# set 512 < max_model_len <= 2048
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=1024,
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
assert model_config.max_model_len == 1024
# set max_model_len > 2048
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=4096,
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
assert model_config.max_model_len == 4096
@pytest.mark.parametrize("model_info", MODELS)
def test_use_rope_scaling_legal(model_info, vllm_runner):
hf_overrides = {
"rope_parameters": {
"rope_theta": rope_theta,
"rope_type": "yarn",
"factor": factor,
"original_max_position_embeddings": original_max_position_embeddings,
},
"max_model_len": max_model_len,
}
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=None,
hf_overrides=hf_overrides,
):
pass
@@ -1,167 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from tests.models.utils import softmax
from vllm.config import PoolerConfig
@pytest.mark.parametrize(
"model",
["jason9693/Qwen2.5-1.5B-apeach", "papluca/xlm-roberta-base-language-detection"],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_classify_models_using_activation(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=False),
) as vllm_model:
wo_activation_out = vllm_model.classify(example_prompts)
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=True),
) as vllm_model:
w_activation_out = vllm_model.classify(example_prompts)
for wo_activation, w_activation in zip(wo_activation_out, w_activation_out):
wo_activation = torch.tensor(wo_activation)
w_activation = torch.tensor(w_activation)
assert not torch.allclose(wo_activation, w_activation, atol=1e-2), (
"pooler_config is not working"
)
assert torch.allclose(
softmax(wo_activation), w_activation, 1e-3 if dtype == "float" else 1e-2
)
@pytest.mark.parametrize(
"model",
[
"intfloat/multilingual-e5-small",
],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_embed_models_using_normalize(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=False),
) as vllm_model:
wo_normalize = torch.tensor(vllm_model.embed(example_prompts))
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=True),
) as vllm_model:
w_normalize = torch.tensor(vllm_model.embed(example_prompts))
assert not torch.allclose(wo_normalize, w_normalize, atol=1e-2), (
"pooler_config normalize is not working"
)
assert torch.allclose(
F.normalize(wo_normalize, p=2, dim=-1), w_normalize, atol=1e-2
), "w_normal should be close to normal(wo_normal)."
@pytest.mark.parametrize(
"model",
[
"internlm/internlm2-1_8b-reward",
],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_reward_models_using_activation(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(
model,
max_model_len=1024,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=False),
) as vllm_model:
wo_activation = vllm_model.token_classify(example_prompts)
with vllm_runner(
model,
max_model_len=1024,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=True),
) as vllm_model:
w_activation = vllm_model.token_classify(example_prompts)
for wo, w in zip(wo_activation, w_activation):
wo = torch.tensor(wo)
w = torch.tensor(w)
assert not torch.allclose(wo, w, atol=1e-2), (
"pooler_config activation is not working"
)
assert torch.allclose(softmax(wo), w, atol=1e-2), (
"w_activation should be close to activation(wo_activation)."
)
@pytest.mark.parametrize(
"model",
[
"intfloat/multilingual-e5-small",
],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_multi_vector_retrieval_models_using_normalize(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=False, task="token_embed"),
) as vllm_model:
wo_normalize = vllm_model.token_embed(example_prompts)
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=True, task="token_embed"),
) as vllm_model:
w_normalize = vllm_model.token_embed(example_prompts)
for wo, w in zip(wo_normalize, w_normalize):
assert not torch.allclose(wo, w, atol=1e-2), (
"pooler_config normalize is not working"
)
assert torch.allclose(F.normalize(wo, p=2, dim=-1), w, atol=1e-2), (
"w_normal should be close to normal(wo_normal)."
)
@@ -1,179 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from typing import TYPE_CHECKING
import pytest
import torch
import torch.nn.functional as F
from transformers import AutoModel
from vllm.platforms import current_platform
from vllm.utils.mem_constants import MiB_bytes
from ....conftest import HfRunner
from ....utils import VLLM_PATH
from ...registry import HF_EXAMPLE_MODELS
if TYPE_CHECKING:
from _typeshed import StrPath
FIXTURES_PATH = VLLM_PATH / "tests/models/fixtures"
assert FIXTURES_PATH.exists()
FIXTURE_REWARD_RESULT = {
"Qwen/Qwen2.5-Math-PRM-7B": FIXTURES_PATH / "qwen2_5_math_prm_reward_step.json",
}
@pytest.fixture
def math_step_prompts():
# ruff: noqa: E501
data = {
"system": "Please reason step by step, and put your final answer within \\boxed{}. ",
"query": "Sue lives in a fun neighborhood. One weekend, the neighbors decided to play a prank on Sue. On Friday morning, the neighbors placed 18 pink plastic flamingos out on Sue's front yard. On Saturday morning, the neighbors took back one third of the flamingos, painted them white, and put these newly painted white flamingos back out on Sue's front yard. Then, on Sunday morning, they added another 18 pink plastic flamingos to the collection. At noon on Sunday, how many more pink plastic flamingos were out than white plastic flamingos?",
"response": [
"To find out how many more pink plastic flamingos were out than white plastic flamingos at noon on Sunday, we can break down the problem into steps. First, on Friday, the neighbors start with 18 pink plastic flamingos.",
"On Saturday, they take back one third of the flamingos. Since there were 18 flamingos, (1/3 \\times 18 = 6) flamingos are taken back. So, they have (18 - 6 = 12) flamingos left in their possession. Then, they paint these 6 flamingos white and put them back out on Sue's front yard. Now, Sue has the original 12 pink flamingos plus the 6 new white ones. Thus, by the end of Saturday, Sue has (12 + 6 = 18) pink flamingos and 6 white flamingos.",
"On Sunday, the neighbors add another 18 pink plastic flamingos to Sue's front yard. By the end of Sunday morning, Sue has (18 + 18 = 36) pink flamingos and still 6 white flamingos.",
"To find the difference, subtract the number of white flamingos from the number of pink flamingos: (36 - 6 = 30). Therefore, at noon on Sunday, there were 30 more pink plastic flamingos out than white plastic flamingos. The answer is (\\boxed{30}).",
],
}
answer = "<extra_0>".join(data["response"]) + "<extra_0>"
prompt = f"<im_start>system\n{data['system']}<im_end>\n<im_start>user\n{data['query']}<im_end>\n<im_start>assistant\n{answer}<im_end><|endoftext|>"
return [prompt]
def step_reward_patch_hf_model(hf_model: HfRunner):
# Patch the hf_runner to use the step reward function
def make_step_rewards(
logits: torch.Tensor, token_masks: torch.Tensor
) -> list[list[float]]:
probabilities = F.softmax(logits, dim=-1)
probabilities = probabilities * token_masks.unsqueeze(-1)
all_scores_res: list[list[float]] = []
for i in range(probabilities.size(0)):
sample = probabilities[i] # seq_len, num_labels
positive_probs = sample[sample != 0].view(-1, 2)
non_zero_elements_list = positive_probs.cpu().tolist()
all_scores_res.append(non_zero_elements_list)
return all_scores_res
def reward(prompts: list[str]) -> list[list[float]]:
input_ids = hf_model.tokenizer(prompts, return_tensors="pt").input_ids
input_ids = hf_model.wrap_device(input_ids)
outputs = hf_model.model(input_ids=input_ids)
step_sep_id = hf_model.tokenizer.encode("<extra_0>")[0]
token_masks = input_ids == step_sep_id
return make_step_rewards(outputs[0], token_masks)
hf_model.reward = reward # type: ignore[attr-defined]
return hf_model
def dump_reward_outputs(outputs: list[list[float]], filename: "StrPath"):
with open(filename, "w", encoding="utf-8") as f:
json.dump(outputs, f)
def load_reward_outputs(filename: "StrPath") -> list[list[float]]:
with open(filename, encoding="utf-8") as f:
return json.load(f)
@pytest.mark.parametrize(
"model",
[
pytest.param(
"Qwen/Qwen2.5-Math-PRM-7B",
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
),
],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_prm_models(
hf_runner,
vllm_runner,
math_step_prompts,
monkeypatch,
model: str,
dtype: str,
) -> None:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_transformers_version(on_fail="skip")
if current_platform.is_cpu():
pytest.skip("CPU only supports V1")
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
max_model_len=1024,
dtype=dtype,
kv_cache_memory_bytes=64 * MiB_bytes,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = vllm_model.token_classify(math_step_prompts)
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
hf_model = step_reward_patch_hf_model(hf_model)
hf_outputs = hf_model.reward(math_step_prompts)
dump_reward_outputs(
hf_outputs,
FIXTURE_REWARD_RESULT[model],
)
# check logits difference
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = torch.tensor(hf_output).float()
vllm_output = torch.tensor(vllm_output).float()
assert torch.allclose(hf_output, vllm_output, 1.5e-2)
@pytest.mark.parametrize(
"model",
[
pytest.param(
"Qwen/Qwen2.5-Math-PRM-7B",
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
),
],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_prm_models_with_golden_outputs(
vllm_runner,
math_step_prompts,
monkeypatch,
model: str,
dtype: str,
) -> None:
if not FIXTURE_REWARD_RESULT.get(model):
pytest.skip(f"No available golden outputs for {model}.")
use_v2_model_runner = not current_platform.is_cpu()
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2_model_runner else "0")
with vllm_runner(
model,
max_model_len=1024,
dtype=dtype,
kv_cache_memory_bytes=64 * MiB_bytes,
) as vllm_model:
assert (
vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
is use_v2_model_runner
)
vllm_outputs = vllm_model.token_classify(math_step_prompts)
golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
# check logits difference
for golden_output, vllm_output in zip(golden_outputs, vllm_outputs):
golden_output = torch.tensor(golden_output).float()
vllm_output = torch.tensor(vllm_output).float()
assert torch.allclose(golden_output, vllm_output, 1.5e-2)
@@ -1,281 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock, patch
import numpy as np
import pytest
import torch
import torch.nn as nn
from vllm.model_executor.models.bert import (
BertMLMHead,
SPLADESparsePooler,
)
from vllm.pooling_params import PoolingParams
from vllm.v1.pool.late_interaction_runner import LateInteractionRunner
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
from vllm.v1.worker.gpu.states import RequestState
# ---------------------------------------------------------------------
# Functional test: SPLADE formula correctness (no HF download needed)
# ---------------------------------------------------------------------
@pytest.mark.parametrize("B,T,H,V", [(2, 3, 5, 7)])
@torch.inference_mode
def test_splade_pooler_matches_reference_formula(B, T, H, V):
"""Ensure SPLADESparsePooler forward() matches the mathematical formula:
log1p(relu(logits)) -> max over sequence length (after masking)."""
torch.manual_seed(0)
# Prepare [B] sequences of shape [T, H]
hs_list = [torch.randn(T, H) for _ in range(B)]
hs_tenser = torch.cat(hs_list)
# Simulate PoolingMetadata (only required fields)
prompt_lens = [T, T - 1]
prompt_lens_tenser = torch.tensor(prompt_lens, dtype=torch.int32)
token_ids = torch.tensor(
[
[101, 5, 102], # Batch 0: [CLS], token, [SEP]
[101, 6, 6], # Batch 1: [CLS], token, token (last token ignored)
],
dtype=torch.long,
)
meta = PoolingMetadata(
prompt_lens=prompt_lens_tenser,
prompt_token_ids=token_ids,
prompt_token_ids_cpu=token_ids,
pooling_params=[PoolingParams(task="embed")] * B,
pooling_states=[PoolingStates() for _ in range(B)],
)
# MLM head (prefer BertMLMHead, fallback to Linear if unavailable)
try:
mlm_head = BertMLMHead(hidden_size=H, vocab_size=V, layer_norm_eps=1e-12)
except Exception:
mlm_head = nn.Linear(H, V, bias=True)
# Forward pass through SPLADE pooler
pooler = SPLADESparsePooler(mlm_head=mlm_head, pooling="max", remove_cls_sep=True)
pooled = pooler(hidden_states=hs_tenser, pooling_metadata=meta) # list of [V]
# Basic output checks
assert isinstance(pooled, torch.Tensor) and len(pooled) == B
for vec in pooled:
assert vec.shape == (V,)
assert torch.isfinite(vec).all()
assert (vec >= 0).all(), "SPLADE outputs must be non-negative."
# Reference implementation for comparison
def ref_one(hs: torch.Tensor, L: int, tid_row: torch.Tensor) -> torch.Tensor:
keep = torch.ones(L, dtype=torch.bool)
if L > 0 and tid_row[0].item() == 101: # remove CLS
keep[0] = False
if L > 0 and tid_row[L - 1].item() == 102: # remove SEP
keep[L - 1] = False
valid = hs[:L][keep[:L]]
if valid.numel() == 0:
return torch.zeros(V, dtype=torch.float32)
logits = mlm_head(valid) # [L', V]
scores = torch.log1p(torch.relu(logits)) # [L', V]
return scores.max(dim=0).values.to(torch.float32)
torch.testing.assert_close(
pooled[0],
ref_one(hs_list[0], prompt_lens[0], token_ids[0]),
rtol=1e-4,
atol=1e-4,
)
torch.testing.assert_close(
pooled[1],
ref_one(hs_list[1], prompt_lens[1], token_ids[1]),
rtol=1e-4,
atol=1e-4,
)
def test_pooling_runner_gathers_required_token_ids() -> None:
runner = PoolingRunner.__new__(PoolingRunner)
pooling_params = PoolingParams(task="embed", requires_token_ids=True)
runner.pooling_params = {1: pooling_params, 3: pooling_params}
runner.pooling_states = {1: PoolingStates(), 3: PoolingStates()}
runner.prompt_token_ids = {
1: torch.tensor([101, 102]),
3: torch.tensor([101, 11, 102]),
}
input_batch = MagicMock(spec=InputBatch)
input_batch.idx_mapping_np = np.array([3, 1], dtype=np.int32)
input_batch.num_reqs = 2
req_states = MagicMock(spec=RequestState)
req_states.prompt_len = MagicMock(np=np.array([0, 2, 0, 3], dtype=np.int32))
metadata = runner._get_pooling_metadata(
input_batch, req_states, torch.device("cpu")
)
expected = torch.tensor([[101, 11, 102], [101, 102, 0]])
assert metadata.prompt_token_ids_cpu is not None
assert metadata.prompt_token_ids is None
assert metadata.prompt_token_ids_cpu.is_pinned() is False
torch.testing.assert_close(
metadata.prompt_lens, torch.tensor([3, 2], dtype=torch.int32)
)
torch.testing.assert_close(metadata.prompt_token_ids_cpu, expected)
def test_pooling_runner_stores_only_required_token_ids() -> None:
runner = PoolingRunner.__new__(PoolingRunner)
runner.model = MagicMock()
runner.supported_tasks = frozenset({"embed"})
runner.pooling_params = {}
runner.pooling_states = {}
runner.prompt_token_ids = {}
runner.late_interaction_runner = MagicMock()
runner.add_request("req-1", 1, PoolingParams(task="embed"), [101, 102])
runner.add_request(
"req-2",
2,
PoolingParams(task="embed", requires_token_ids=True),
[101, 11, 102],
)
assert 1 not in runner.prompt_token_ids
torch.testing.assert_close(runner.prompt_token_ids[2], torch.tensor([101, 11, 102]))
def test_pooling_runner_releases_aborted_late_interaction_doc() -> None:
runner = PoolingRunner.__new__(PoolingRunner)
runner.late_interaction_runner = LateInteractionRunner()
query_key = "query-abort"
late_interaction_runner = runner.late_interaction_runner
late_interaction_runner._query_cache[query_key] = torch.ones(2, 4)
late_interaction_runner._query_uses[query_key] = 1
late_interaction_runner._doc_query_keys["doc-req"] = query_key
runner.on_requests_finished({"doc-req"})
assert not late_interaction_runner._query_cache
assert not late_interaction_runner._query_uses
assert not late_interaction_runner._doc_query_keys
def test_encoder_cache_reset_clears_late_interaction_state() -> None:
# Cached query embeddings are only invalidated by weight reloads, which
# reach the runner via reset_encoder_cache. Resetting the multi-modal
# cache is unrelated and must leave them intact.
runner = GPUModelRunner.__new__(GPUModelRunner)
runner.encoder_cache = MagicMock()
runner.pooling_runner = MagicMock()
runner.reset_mm_cache()
runner.encoder_cache.reset_mm_cache.assert_called_once_with()
runner.pooling_runner.clear.assert_not_called()
runner.reset_encoder_cache()
runner.encoder_cache.reset_encoder_cache.assert_called_once_with()
runner.pooling_runner.clear.assert_called_once_with()
def test_pooling_runner_rejects_unsupported_selected_task() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"embed", "plugin"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "encoder_only"
vllm_config.model_config.get_pooling_task.return_value = "plugin"
with (
patch.object(PoolingRunner, "get_supported_tasks", return_value=["embed"]),
pytest.raises(ValueError, match="selects 'plugin'"),
):
PoolingRunner(model, vllm_config)
def test_pooling_runner_supports_encoder_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"token_classify"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "encoder_only"
vllm_config.model_config.get_pooling_task.return_value = "token_classify"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"token_classify"}
def test_pooling_runner_supports_encoder_token_embedding() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"token_embed"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "encoder_only"
vllm_config.model_config.get_pooling_task.return_value = "token_embed"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"token_embed"}
def test_pooling_runner_supports_decoder_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"token_classify"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "decoder"
vllm_config.model_config.get_pooling_task.return_value = "token_classify"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"token_classify"}
def test_pooling_runner_keeps_decoder_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"embed", "token_classify"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "decoder"
vllm_config.model_config.get_pooling_task.return_value = "embed"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"embed", "token_classify"}
def test_pooling_runner_keeps_decoder_token_embedding() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"embed", "token_embed"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "decoder"
vllm_config.model_config.get_pooling_task.return_value = "embed"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"embed", "token_embed"}
def test_pooling_runner_supports_embed_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"embed", "embed&token_classify"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "encoder_only"
vllm_config.model_config.get_pooling_task.return_value = "embed&token_classify"
assert PoolingRunner(model, vllm_config).supported_tasks == {
"embed",
"embed&token_classify",
}
@@ -1,280 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModelForMaskedLM, AutoModelForTokenClassification
from tests.models.registry import HF_EXAMPLE_MODELS
from tests.models.utils import softmax
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@pytest.fixture(autouse=True)
def seed_everything():
"""Seed all random number generators for reproducibility."""
seed = 0
set_random_seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
yield
@pytest.mark.parametrize(
"model",
[
"boltuix/NeuroBERT-NER",
],
)
# The float32 is required for this tiny model to pass the test.
@pytest.mark.parametrize("dtype", ["float"])
@pytest.mark.core_model
@torch.inference_mode
def test_bert_model_runner_v2(
hf_runner,
vllm_runner,
example_prompts,
monkeypatch,
model: str,
dtype: str,
) -> None:
prompt_batches = [[example_prompts[0]], example_prompts]
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_output_batches = [
vllm_model.token_classify(prompts) for prompts in prompt_batches
]
with hf_runner(
model,
dtype=dtype,
auto_cls=AutoModelForTokenClassification,
) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in example_prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
hf_outputs.append(softmax(output.logits[0]))
hf_output_batches = [[hf_outputs[0]], hf_outputs]
# check logits difference
for hf_outputs, vllm_outputs in zip(hf_output_batches, vllm_output_batches):
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
@pytest.mark.parametrize("model", ["disham993/electrical-ner-ModernBERT-base"])
@pytest.mark.parametrize("dtype", ["float"])
@pytest.mark.flaky(reruns=3)
@torch.inference_mode
def test_modernbert_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
# NOTE: https://github.com/vllm-project/vllm/pull/32403
# `disham993/electrical-ner-ModernBERT-base` is a randomly initialized
# model, which can cause numerical precision variance and edge cases.
# We use @flaky(reruns=3) to mitigate intermittent failures.
print(
f"\n[NOTE] Testing {model} (randomly initialized weights) - "
"flaky tolerance enabled due to numerical precision variance."
)
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.token_classify(example_prompts)
with hf_runner(
model,
dtype=dtype,
auto_cls=AutoModelForTokenClassification,
) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in example_prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
hf_outputs.append(softmax(output.logits[0]))
# check logits difference
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
torch.testing.assert_close(hf_output, vllm_output, atol=6e-2, rtol=1e-3)
@pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"])
@pytest.mark.parametrize("dtype", ["float"])
@pytest.mark.flaky(reruns=3)
@torch.inference_mode
def test_xlm_roberta_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.token_classify(example_prompts)
with hf_runner(
model,
dtype=dtype,
auto_cls=AutoModelForTokenClassification,
) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in example_prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
hf_outputs.append(softmax(output.logits[0]))
# check logits difference
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
PRIVACY_FILTER_PROMPTS = [
"My name is Harry Potter.",
"Email me at harry.potter@hogwarts.edu.",
"Call me on +44 20 7946 0958 tomorrow.",
"My account number is 12345678 and the API key is sk-live-abc123def456.",
"I live at 4 Privet Drive, Little Whinging.",
"Visit https://example.com/profile/harry for more info.",
"We met on 12 January 2024.",
]
@pytest.mark.skipif(
current_platform.is_rocm(),
reason="Workspace growth allocation issue on ROCm. See https://github.com/vllm-project/vllm/issues/48510",
)
@pytest.mark.parametrize("model", ["openai/privacy-filter"])
@pytest.mark.parametrize("dtype", ["bfloat16"])
@torch.inference_mode
def test_openai_privacy_filter(
hf_runner,
vllm_runner,
model: str,
dtype: str,
) -> None:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_transformers_version(on_fail="skip")
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.token_classify(PRIVACY_FILTER_PROMPTS)
with hf_runner(
model,
dtype=dtype,
auto_cls=AutoModelForTokenClassification,
) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in PRIVACY_FILTER_PROMPTS:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
hf_outputs.append(softmax(output.logits[0]))
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
torch.testing.assert_close(hf_output, vllm_output, atol=0.1, rtol=1e-2)
@pytest.mark.parametrize("model", ["bd2lcco/Qwen3-0.6B-finetuned"])
@pytest.mark.parametrize("dtype", ["float"])
@pytest.mark.flaky(reruns=3)
@torch.inference_mode
def test_auto_conversion(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.token_classify(example_prompts)
with hf_runner(
model, dtype=dtype, auto_cls=AutoModelForTokenClassification
) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in example_prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
hf_outputs.append(softmax(output.logits[0]))
# check logits difference
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
assert torch.allclose(hf_output, vllm_output, atol=1e-2)
@pytest.mark.parametrize(
"model",
[
# Original Google checkpoint: legacy `gamma`/`beta` LayerNorm names, an
# NSP head (`cls.seq_relationship.*`) and a decoder tied to the input
# embeddings (no explicit decoder weight in the checkpoint).
"google-bert/bert-base-uncased",
],
)
@pytest.mark.parametrize("dtype", ["float"])
@torch.inference_mode
def test_bert_for_masked_lm(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
# BertForMaskedLM exposes its MLM head as a token-level pooling task; the
# head applies softmax over the vocabulary, so each output row is a
# distribution (matching HF's softmax(logits) below).
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.token_classify(example_prompts)
# Run hf_runner reference with "highest" fp32 precision to match
# default behvior of vLLM. This is needed on ROCm since the
# pooling tests set matmul precision to "high" in conftest.py
prev_matmul_precision = torch.get_float32_matmul_precision()
torch.set_float32_matmul_precision("highest")
with hf_runner(
model,
dtype=dtype,
auto_cls=AutoModelForMaskedLM,
) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in example_prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
hf_outputs.append(softmax(output.logits[0]))
torch.set_float32_matmul_precision(prev_matmul_precision)
# Compare the per-token vocabulary distributions position by position.
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
assert hf_output.shape == vllm_output.shape
assert torch.equal(hf_output.argmax(dim=-1), vllm_output.argmax(dim=-1))
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
@@ -1,70 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.exceptions import VLLMValidationError
MODEL_NAME = "sentence-transformers/all-MiniLM-L12-v2"
max_model_len = 128
input_str = """Immerse yourself in the enchanting chronicle of calculus, a
mathematical domain that has radically transformed our comprehension of
change and motion. Despite its roots in ancient civilizations, the
formal birth of calculus predominantly occurred in the 17th century,
primarily under the influential guidance of Sir Isaac Newton and Gottfried
Wilhelm Leibniz. The earliest traces of calculus concepts are found in
ancient Greek mathematics,most notably in the works of Eudoxus and
Archimedes, around 300 BCE. They utilized the 'method of exhaustion'—a
technique for computing areas and volumes through the use of finite sums.
This methodology laid crucial foundational work for integral calculus.
In the 17th century, both Newton and Leibniz independently pioneered
calculus, each contributing unique perspectives that would shape this new
field."""
@pytest.fixture(scope="module")
def vllm_model(vllm_runner):
with vllm_runner(
MODEL_NAME, runner="pooling", max_model_len=max_model_len
) as model:
yield model
def test_smaller_truncation_size(
vllm_model,
):
truncate_prompt_tokens = 10
vllm_output = vllm_model.llm.embed(
input_str,
tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
)
prompt_tokens = vllm_output[0].prompt_token_ids
assert len(prompt_tokens) == truncate_prompt_tokens
def test_max_truncation_size(vllm_model):
truncate_prompt_tokens = -1
vllm_output = vllm_model.llm.embed(
input_str,
tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
)
prompt_tokens = vllm_output[0].prompt_token_ids
# Pooling requests emit an embedding rather than generated tokens, so -1
# truncates to the full context length.
assert len(prompt_tokens) == max_model_len
def test_bigger_truncation_size(vllm_model):
truncate_prompt_tokens = max_model_len + 1
with pytest.raises(VLLMValidationError):
vllm_model.llm.embed(
input_str,
tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
)