[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,240 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CompilationConfig, VllmConfig
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.models import gemma
|
||||
from vllm.model_executor.models.gemma3n import (
|
||||
Gemma3nTextModel,
|
||||
_kv_sharing_weights_mapper,
|
||||
)
|
||||
from vllm.model_executor.models.gemma4 import (
|
||||
Gemma4ForCausalLM,
|
||||
_gemma4_layer_weights_mapper,
|
||||
)
|
||||
from vllm.model_executor.models.gemma4_dspark import (
|
||||
Gemma4DSparkForCausalLM,
|
||||
Gemma4DSparkModel,
|
||||
)
|
||||
|
||||
MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
@pytest.mark.parametrize(
|
||||
"enabled,has_weights,with_markov",
|
||||
[
|
||||
pytest.param(True, True, False, id="hidden-only"),
|
||||
pytest.param(True, True, True, id="with-markov"),
|
||||
pytest.param(True, False, True, id="missing-weights"),
|
||||
pytest.param(False, True, True, id="disabled-head"),
|
||||
],
|
||||
)
|
||||
def test_gemma4_dspark_loads_confidence_head(
|
||||
monkeypatch, enabled, has_weights, with_markov
|
||||
) -> None:
|
||||
"""Use checkpoint confidence parameters, or disable an unavailable head."""
|
||||
config = SimpleNamespace(
|
||||
vocab_size=64,
|
||||
hidden_size=8,
|
||||
target_layer_ids=[0, 1],
|
||||
num_hidden_layers=0,
|
||||
rms_norm_eps=1e-6,
|
||||
markov_rank=4,
|
||||
enable_confidence_head=enabled,
|
||||
confidence_head_with_markov=with_markov,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
model_config=SimpleNamespace(dtype=torch.bfloat16),
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(hf_config=config),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(Gemma4DSparkModel, "_build_fused_kv_buffers", lambda _: None)
|
||||
model = Gemma4DSparkForCausalLM(vllm_config=cast(VllmConfig, vllm_config))
|
||||
width = config.hidden_size + (config.markov_rank if with_markov else 0)
|
||||
weight = torch.arange(width, dtype=torch.bfloat16).reshape(1, -1) / 16
|
||||
bias = torch.tensor([-0.5], dtype=torch.bfloat16)
|
||||
loaded = model.load_weights(
|
||||
[("confidence_head.proj.weight", weight), ("confidence_head.proj.bias", bias)]
|
||||
if has_weights
|
||||
else []
|
||||
)
|
||||
if not (enabled and has_weights):
|
||||
assert model.model.confidence_head is None
|
||||
assert not loaded
|
||||
return
|
||||
|
||||
assert loaded == {
|
||||
"model.confidence_head.proj.weight",
|
||||
"model.confidence_head.proj.bias",
|
||||
}
|
||||
assert model.model.confidence_head.proj.weight.dtype == torch.float32
|
||||
hidden = torch.full((3, config.hidden_size), 0.25, dtype=torch.bfloat16)
|
||||
markov = torch.full((3, config.markov_rank), -0.5, dtype=torch.bfloat16)
|
||||
inputs = torch.cat([hidden, markov], dim=-1) if with_markov else hidden
|
||||
expected = (inputs.float() @ weight.float().T + bias.float()).sigmoid().squeeze(-1)
|
||||
torch.testing.assert_close(model.compute_confidence(hidden, markov), expected)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
|
||||
"""A physical LM head must load after checkpoint-driven untying."""
|
||||
|
||||
class StubGemmaModel(torch.nn.Module):
|
||||
def __init__(self, *, vllm_config, prefix):
|
||||
super().__init__()
|
||||
self.embed_tokens = VocabParallelEmbedding(4, 2)
|
||||
self.make_empty_intermediate_tensors = None
|
||||
|
||||
monkeypatch.setattr(gemma, "GemmaModel", StubGemmaModel)
|
||||
config = SimpleNamespace(
|
||||
vocab_size=4,
|
||||
hidden_size=2,
|
||||
tie_word_embeddings=False,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=config),
|
||||
quant_config=None,
|
||||
)
|
||||
model = gemma.GemmaForCausalLM(vllm_config=cast(VllmConfig, vllm_config))
|
||||
embedding_weight = torch.full((4, 2), 1.0)
|
||||
lm_head_weight = torch.full((4, 2), 2.0)
|
||||
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
("model.embed_tokens.weight", embedding_weight),
|
||||
("lm_head.weight", lm_head_weight),
|
||||
]
|
||||
)
|
||||
|
||||
assert loaded == {"model.embed_tokens.weight", "lm_head.weight"}
|
||||
assert torch.equal(model.model.embed_tokens.weight[:4], embedding_weight)
|
||||
assert torch.equal(model.lm_head.weight[:4], lm_head_weight)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_gemma4_attention_mapper() -> None:
|
||||
"""Layers with a qkv_proj pack q/k/v; `attention_k_eq_v` full-attention
|
||||
layers also load K as the V shard, and leave any v_proj such a checkpoint
|
||||
ships unmapped so it fails the load rather than silently overwriting V;
|
||||
KV-shared layers keep q_proj and drop the K/V tensors original checkpoints
|
||||
still ship for them."""
|
||||
config = SimpleNamespace(
|
||||
num_hidden_layers=3,
|
||||
num_kv_shared_layers=1,
|
||||
attention_k_eq_v=True,
|
||||
layer_types=["sliding_attention", "full_attention", "sliding_attention"],
|
||||
)
|
||||
weights = [
|
||||
(f"model.layers.{i}.self_attn.{tensor}.weight", torch.full((2, 2), i + 1.0))
|
||||
for i in range(3)
|
||||
for tensor in ("q_proj", "k_proj", "k_norm")
|
||||
] + [
|
||||
("model.layers.0.mlp.up_proj.weight", torch.empty(0)),
|
||||
("model.layers.1.self_attn.v_proj.weight", torch.empty(0)),
|
||||
]
|
||||
|
||||
mapper = _gemma4_layer_weights_mapper(config)
|
||||
mapped = list(mapper.apply(weights))
|
||||
|
||||
assert [(name, getattr(w, "shard_id", None)) for name, w in mapped] == [
|
||||
("model.layers.0.self_attn.qkv_proj.weight", "q"),
|
||||
("model.layers.0.self_attn.qkv_proj.weight", "k"),
|
||||
("model.layers.0.self_attn.k_norm.weight", None),
|
||||
("model.layers.1.self_attn.qkv_proj.weight", "q"),
|
||||
("model.layers.1.self_attn.qkv_proj.weight", "k"),
|
||||
("model.layers.1.self_attn.qkv_proj.weight", "v"),
|
||||
("model.layers.1.self_attn.k_norm.weight", None),
|
||||
("model.layers.2.self_attn.q_proj.weight", None),
|
||||
("model.layers.0.mlp.gate_up_proj.weight", 1),
|
||||
("model.layers.1.self_attn.v_proj.weight", None),
|
||||
]
|
||||
k_weight, v_weight = weights[4][1], mapped[5][1]
|
||||
assert torch.equal(v_weight, k_weight) and v_weight is not k_weight
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_gemma4_expert_names_strip_language_model_prefix() -> None:
|
||||
"""The text-only path reuses the conditional wrapper's checkpoint naming,
|
||||
so fused and per-expert tensors reach the experts under `model.*`."""
|
||||
prefix = "model.language_model.layers.0."
|
||||
weights = [
|
||||
(prefix + name, torch.empty(0))
|
||||
for name in (
|
||||
"experts.gate_up_proj",
|
||||
"experts.3.down_proj.weight_packed",
|
||||
"router.per_expert_scale",
|
||||
)
|
||||
]
|
||||
|
||||
mapped = [
|
||||
(name, getattr(w, "shard_id", None))
|
||||
for name, w in Gemma4ForCausalLM.hf_to_vllm_mapper.apply(weights)
|
||||
]
|
||||
|
||||
assert mapped == [
|
||||
("model.layers.0.experts.gate_up_proj", None),
|
||||
("model.layers.0.experts.3.down_proj.weight_packed", None),
|
||||
("model.layers.0.router.per_expert_scale", None),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_gemma3n_kv_shared_layer_mapper() -> None:
|
||||
"""Only non-shared layers pack q/k/v into qkv_proj; KV-shared layers keep
|
||||
q_proj and drop the redundant K/V tensors original checkpoints ship."""
|
||||
config = SimpleNamespace(num_hidden_layers=4, num_kv_shared_layers=2)
|
||||
mapper = Gemma3nTextModel.hf_to_vllm_mapper | _kv_sharing_weights_mapper(config)
|
||||
weights = [
|
||||
(f"layers.{i}.self_attn.{tensor}.weight", torch.empty(0))
|
||||
for i in (1, 3)
|
||||
for tensor in ("q_proj", "k_proj", "v_proj", "k_norm", "o_proj")
|
||||
]
|
||||
|
||||
mapped = [
|
||||
(name, getattr(weight, "shard_id", None))
|
||||
for name, weight in mapper.apply(weights)
|
||||
]
|
||||
|
||||
assert mapped == [
|
||||
("layers.1.self_attn.qkv_proj.weight", "q"),
|
||||
("layers.1.self_attn.qkv_proj.weight", "k"),
|
||||
("layers.1.self_attn.qkv_proj.weight", "v"),
|
||||
("layers.1.self_attn.k_norm.weight", None),
|
||||
("layers.1.self_attn.o_proj.weight", None),
|
||||
("layers.3.self_attn.q_proj.weight", None),
|
||||
("layers.3.self_attn.o_proj.weight", None),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None:
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
load_format="dummy",
|
||||
) as llm:
|
||||
if model == "google/gemma-3-4b-it":
|
||||
normalizers = llm.llm.collective_rpc(
|
||||
lambda self: (
|
||||
self.model_runner.model.language_model.model.normalizer.cpu().item()
|
||||
) # noqa: E501
|
||||
)
|
||||
config = llm.llm.llm_engine.model_config.hf_config.text_config
|
||||
else:
|
||||
normalizers = llm.llm.collective_rpc(
|
||||
lambda self: self.model_runner.model.model.normalizer.cpu().item()
|
||||
)
|
||||
config = llm.llm.llm_engine.model_config.hf_config
|
||||
assert np.allclose(normalizers, config.hidden_size**0.5, rtol=2e-3)
|
||||
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
from transformers import GraniteConfig
|
||||
|
||||
from vllm.model_executor.models.granite import granite_layer_attn_params
|
||||
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
MODELS = [
|
||||
# TODO(sang): Sliding window should be tested separately.
|
||||
"ibm/PowerLM-3b",
|
||||
"ibm/PowerMoE-3b",
|
||||
"ibm-granite/granite-swash-2b",
|
||||
"ibm-granite/granite-swash-3b-a600m",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.cpu_model
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
with hf_runner(model, dtype=dtype) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
with vllm_runner(model, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
def test_granite_swa_features_are_off_without_swa_config():
|
||||
"""A plain Granite config must not pick up sliding windows or sinks."""
|
||||
config = GraniteConfig(num_hidden_layers=4)
|
||||
theta = config.rope_parameters["rope_theta"]
|
||||
|
||||
assert [granite_layer_attn_params(config, i) for i in range(4)] == [
|
||||
(None, theta, False)
|
||||
] * 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"attention_sinks, expected_sink", [(None, True), (False, False)]
|
||||
)
|
||||
def test_granite_swa_features_resolve_per_layer(attention_sinks, expected_sink):
|
||||
"""`layer_types`/`layer_rope_theta` apply per layer; theta 0 means NoPE."""
|
||||
kwargs = {} if attention_sinks is None else {"attention_sinks": attention_sinks}
|
||||
config = GraniteConfig(
|
||||
num_hidden_layers=3,
|
||||
sliding_window=128,
|
||||
layer_types=["full_attention", "sliding_attention", "sliding_attention"],
|
||||
layer_rope_theta=[10000.0, 0.0, 1000000.0],
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
assert [granite_layer_attn_params(config, i) for i in range(3)] == [
|
||||
(None, 10000.0, expected_sink),
|
||||
(128, 0.0, expected_sink),
|
||||
(128, 1000000.0, expected_sink),
|
||||
]
|
||||
@@ -1,109 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....utils import large_gpu_test
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
MODELS = [
|
||||
"microsoft/Phi-3.5-MoE-instruct",
|
||||
]
|
||||
|
||||
|
||||
def test_phimoe_routing_function():
|
||||
from vllm.model_executor.models.phimoe import phimoe_routing_function
|
||||
|
||||
test_case = {
|
||||
0: {
|
||||
"hidden_states": torch.tensor(
|
||||
[1, 2, 3, 4, 5, 6, 7, 8], dtype=torch.float32, requires_grad=False
|
||||
).view(4, 2),
|
||||
"gating_output": torch.tensor(
|
||||
[0.1, 0.2, 0.3, 0.4], dtype=torch.float32, requires_grad=False
|
||||
),
|
||||
"topk": 2,
|
||||
"renormalize": False,
|
||||
},
|
||||
1: {
|
||||
"hidden_states": torch.tensor(
|
||||
[1, 2, 3, 4, 5, 6, 7, 8], dtype=torch.float32, requires_grad=False
|
||||
).view(4, 2),
|
||||
"gating_output": torch.tensor(
|
||||
[0.4, 0.2, 0.3, 0.4], dtype=torch.float32, requires_grad=False
|
||||
),
|
||||
"topk": 2,
|
||||
"renormalize": False,
|
||||
},
|
||||
}
|
||||
|
||||
ground_truth = {
|
||||
0: {
|
||||
"topk_weights": torch.tensor(
|
||||
[1.0, 1.0], dtype=torch.float32, requires_grad=False
|
||||
),
|
||||
"topk_ids": torch.tensor([3, 2], dtype=torch.long, requires_grad=False),
|
||||
},
|
||||
1: {
|
||||
"topk_weights": torch.tensor(
|
||||
[0.5, 1.0], dtype=torch.float32, requires_grad=False
|
||||
),
|
||||
"topk_ids": torch.tensor([0, 3], dtype=torch.long, requires_grad=False),
|
||||
},
|
||||
}
|
||||
|
||||
for test_id in test_case:
|
||||
topk_weights, topk_ids = phimoe_routing_function(**test_case[test_id])
|
||||
assert torch.allclose(topk_weights, ground_truth[test_id]["topk_weights"])
|
||||
assert torch.equal(topk_ids, ground_truth[test_id]["topk_ids"])
|
||||
|
||||
|
||||
# There is a known issue that triggers `AttributeError: 'DynamicCache'
|
||||
# object has no attribute 'seen_tokens'` when running:
|
||||
# `tests/models/language/generation/test_phimoe.py::test_models
|
||||
# [5-64-bfloat16-microsoft/Phi-3.5-MoE-instruct]`
|
||||
# This issue is being investigated and tracked in:
|
||||
# https://huggingface.co/microsoft/Phi-3.5-MoE-instruct/discussions/58
|
||||
# It is platform-agnostic. Therefore, we skip this test on all platforms for now.
|
||||
@pytest.mark.skip(
|
||||
reason="Skipping due to known issue: "
|
||||
"'DynamicCache' object has no attribute 'seen_tokens'. See: "
|
||||
"https://huggingface.co/microsoft/Phi-3.5-MoE-instruct/discussions/58 "
|
||||
"for details.",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
condition=current_platform.is_cpu(),
|
||||
reason="This test takes a lot time to run on CPU, "
|
||||
"and vllm CI's disk space is not enough for this model.",
|
||||
)
|
||||
@large_gpu_test(min_gb=80)
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
with hf_runner(model, dtype=dtype) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
with vllm_runner(model, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
@@ -1,43 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pytest configuration for vLLM language generation tests."""
|
||||
|
||||
import gc
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def pytest_sessionstart(session):
|
||||
"""Configure ROCm-specific settings before test session starts."""
|
||||
if current_platform.is_rocm():
|
||||
torch.set_float32_matmul_precision("high")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def release_gpu_memory_between_tests():
|
||||
"""Reclaim cached GPU memory between tests in this shard.
|
||||
|
||||
These tests start many vLLM/HF engines sequentially in one pytest
|
||||
process; without explicit reclamation between tests, VRAM fragmentation
|
||||
accumulates and later tests fail at engine startup ("Engine core
|
||||
initialization failed").
|
||||
"""
|
||||
yield
|
||||
gc.collect()
|
||||
if torch.accelerator.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
if current_platform.is_rocm():
|
||||
return
|
||||
try:
|
||||
wait_for_memory_to_settle()
|
||||
except ValueError as e:
|
||||
# Longer-lived (class/module-scoped) engine fixtures may legitimately
|
||||
# still hold VRAM; nothing further to reclaim in that case.
|
||||
logger.info("Failed to clean GPU memory: %s", e)
|
||||
@@ -1,90 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
|
||||
|
||||
def float_embeddings(response: EmbeddingResponse) -> list[list[float]]:
|
||||
embeddings = []
|
||||
for d in response.data:
|
||||
assert isinstance(d.embedding, list)
|
||||
embeddings.append(d.embedding)
|
||||
return embeddings
|
||||
|
||||
|
||||
def run_embedding_correctness_test(
|
||||
hf_model: "HfRunner",
|
||||
inputs: list[str],
|
||||
vllm_outputs: Sequence[list[float]],
|
||||
dimensions: int | None = None,
|
||||
):
|
||||
hf_outputs = hf_model.encode(inputs)
|
||||
if dimensions:
|
||||
hf_outputs = matryoshka_fy(hf_outputs, dimensions)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
def correctness_test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info: EmbedModelInfo,
|
||||
example_prompts,
|
||||
vllm_extra_kwargs=None,
|
||||
hf_model_callback=None,
|
||||
):
|
||||
pytest.skip("Debug only, ci prefers to use mteb test.")
|
||||
|
||||
# The example_prompts has ending "\n", for example:
|
||||
# "Write a short story about a robot that dreams for the first time.\n"
|
||||
# sentence_transformers will strip the input texts, see:
|
||||
# https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
|
||||
# This makes the input_ids different between hf_model and vllm_model.
|
||||
# So we need to strip the input texts to avoid test failing.
|
||||
example_prompts = [str(s).strip() for s in example_prompts]
|
||||
|
||||
vllm_extra_kwargs = vllm_extra_kwargs or {}
|
||||
vllm_extra_kwargs["dtype"] = model_info.dtype
|
||||
|
||||
if model_info.hf_overrides is not None:
|
||||
vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides
|
||||
|
||||
with vllm_runner(
|
||||
model_info.name, runner="pooling", max_model_len=None, **vllm_extra_kwargs
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.embed(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model_info.name,
|
||||
dtype=model_info.hf_dtype,
|
||||
is_sentence_transformer=True,
|
||||
) as hf_model:
|
||||
if hf_model_callback is not None:
|
||||
hf_model_callback(hf_model)
|
||||
|
||||
run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
|
||||
|
||||
|
||||
async def run_client_embeddings(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
queries: list[str],
|
||||
instruction: str = "",
|
||||
) -> list[list[float]]:
|
||||
outputs = await client.embeddings.create(
|
||||
model=model_name,
|
||||
input=[instruction + q for q in queries],
|
||||
)
|
||||
return [data.embedding for data in outputs.data]
|
||||
@@ -1,151 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
from tests.models.utils import check_embeddings_close
|
||||
from tests.utils import VLLM_PATH
|
||||
from vllm import TokensPrompt
|
||||
from vllm.config import PoolerConfig
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["Qwen/Qwen3-Embedding-0.6B"],
|
||||
)
|
||||
@torch.inference_mode
|
||||
def test_decoder_token_embed_model_runner_v2(
|
||||
hf_runner, vllm_runner, monkeypatch, model: str
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
chunk_size = 10
|
||||
n_prompt_tokens = [55, 56, 57]
|
||||
token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
pooler_config=PoolerConfig(task="token_embed"),
|
||||
max_model_len=128,
|
||||
max_num_batched_tokens=chunk_size,
|
||||
enforce_eager=True,
|
||||
# `enable_chunked_prefill`: Set to `False` instead of `None` in VllmRunner
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=True,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.token_embed(
|
||||
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
|
||||
)
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
auto_cls=AutoModel,
|
||||
) as hf_model:
|
||||
hf_outputs = []
|
||||
for token_prompt in token_prompts:
|
||||
inputs = hf_model.wrap_device({"input_ids": torch.tensor([token_prompt])})
|
||||
input_ids = inputs["input_ids"]
|
||||
output = hf_model.model(input_ids)
|
||||
hf_outputs.append(output.last_hidden_state.cpu().float()[0])
|
||||
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_output,
|
||||
embeddings_1_lst=vllm_output,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
_RERANKER_HF_OVERRIDES = {
|
||||
"architectures": ["Qwen3ForSequenceClassification"],
|
||||
"classifier_from_token": ["no", "yes"],
|
||||
"is_original_qwen3_reranker": True,
|
||||
}
|
||||
_RERANKER_TEMPLATE = VLLM_PATH / "examples/pooling/score/template/qwen3_reranker.jinja"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["Qwen/Qwen3-Reranker-0.6B"])
|
||||
@torch.inference_mode
|
||||
def test_last_pool_score_chunked_prefill_matches_unchunked(vllm_runner, model: str):
|
||||
"""LAST-pooling score must not depend on whether the prompt is chunked.
|
||||
|
||||
Regression test for a wrong-score bug where, under ``torch.compile`` (i.e.
|
||||
not ``enforce_eager``), a query+document pair long enough to be split across
|
||||
prefill chunks produced a corrupted last-token hidden state and thus a
|
||||
wrong relevance score, while the same input scored correctly unchunked.
|
||||
The existing pooling+chunked-prefill coverage runs ``enforce_eager=True``
|
||||
and so never exercised the compiled path.
|
||||
"""
|
||||
chat_template = _RERANKER_TEMPLATE.read_text()
|
||||
query = "What organelle produces energy in the cell?"
|
||||
# A long, matching document so query + doc exceeds the chunk size below.
|
||||
document = (
|
||||
"The mitochondria is the powerhouse of the cell. It generates most of "
|
||||
"the cell chemical energy through oxidative phosphorylation. "
|
||||
) * 400
|
||||
|
||||
def score_with(max_num_batched_tokens: int) -> float:
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
hf_overrides=_RERANKER_HF_OVERRIDES,
|
||||
max_model_len=16384,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=False,
|
||||
) as vllm_model:
|
||||
return vllm_model.score(query, document, chat_template=chat_template)[0]
|
||||
|
||||
# Chunk size forces the prompt across several prefill chunks; the large
|
||||
# budget keeps it in a single chunk as the reference.
|
||||
chunked = score_with(2048)
|
||||
unchunked = score_with(16384)
|
||||
|
||||
assert chunked == pytest.approx(unchunked, abs=5e-2), (
|
||||
f"chunked score {chunked} diverged from unchunked {unchunked}"
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode
|
||||
def test_sequence_embed_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
|
||||
model = "Qwen/Qwen3-Embedding-0.6B"
|
||||
chunk_size = 10
|
||||
token_prompts = [[1024 + i for i in range(n)] for n in (25, 27)]
|
||||
prompts = [TokensPrompt(prompt_token_ids=t) for t in token_prompts]
|
||||
|
||||
with hf_runner(model, auto_cls=AutoModel) as hf_model:
|
||||
hf_outputs = []
|
||||
for token_prompt in token_prompts:
|
||||
inputs = hf_model.wrap_device({"input_ids": torch.tensor([token_prompt])})
|
||||
output = hf_model.model(inputs["input_ids"])
|
||||
embedding = torch.nn.functional.normalize(
|
||||
output.last_hidden_state.float()[0, -1], dim=0
|
||||
)
|
||||
hf_outputs.append(embedding.cpu().tolist())
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
pooler_config=PoolerConfig(task="embed"),
|
||||
max_model_len=64,
|
||||
max_num_batched_tokens=chunk_size,
|
||||
max_num_seqs=2,
|
||||
gpu_memory_utilization=0.25,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=True,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.embed(prompts)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
@@ -1,113 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForSequenceClassification
|
||||
|
||||
from tests.models.language.pooling.embed_utils import run_embedding_correctness_test
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["jason9693/Qwen2.5-1.5B-apeach"],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_classify_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
# example_prompts is too short for testing prefix_caching
|
||||
example_prompts = [s * 10 for s in example_prompts]
|
||||
|
||||
with vllm_runner(
|
||||
model, max_model_len=512, dtype=dtype, enable_prefix_caching=True
|
||||
) as vllm_model:
|
||||
vllm_config = vllm_model.llm.llm_engine.vllm_config
|
||||
cache_config = vllm_config.cache_config
|
||||
assert cache_config.enable_prefix_caching
|
||||
|
||||
# First Run
|
||||
vllm_model.classify(example_prompts)
|
||||
|
||||
# assert prefix_caching works
|
||||
pooling_outputs = vllm_model.llm.encode(
|
||||
example_prompts, pooling_task="classify"
|
||||
)
|
||||
for output in pooling_outputs:
|
||||
assert output.num_cached_tokens > 0
|
||||
vllm_outputs = [req_output.outputs.data for req_output in pooling_outputs]
|
||||
|
||||
with hf_runner(
|
||||
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.classify(example_prompts)
|
||||
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = torch.tensor(hf_output)
|
||||
vllm_output = torch.tensor(vllm_output)
|
||||
|
||||
assert torch.allclose(
|
||||
hf_output, vllm_output, 1e-3 if dtype == "float" else 1e-2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["Qwen/Qwen3-Embedding-0.6B"],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
):
|
||||
# example_prompts is too short for testing prefix_caching
|
||||
example_prompts = [str(s).strip() * 10 for s in example_prompts]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
max_model_len=None,
|
||||
enable_prefix_caching=True,
|
||||
) as vllm_model:
|
||||
vllm_config = vllm_model.llm.llm_engine.vllm_config
|
||||
cache_config = vllm_config.cache_config
|
||||
assert cache_config.enable_prefix_caching
|
||||
|
||||
# First Run
|
||||
vllm_model.embed(example_prompts)
|
||||
|
||||
# assert prefix_caching works
|
||||
pooling_outputs = vllm_model.llm.encode(example_prompts, pooling_task="embed")
|
||||
for output in pooling_outputs:
|
||||
assert output.num_cached_tokens > 0
|
||||
vllm_outputs = [req_output.outputs.data for req_output in pooling_outputs]
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
is_sentence_transformer=True,
|
||||
) as hf_model:
|
||||
run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"intfloat/e5-small",
|
||||
"Alibaba-NLP/gte-Qwen2-1.5B-instruct", # is_causal == False
|
||||
"papluca/xlm-roberta-base-language-detection",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_non_causal_models(
|
||||
hf_runner, vllm_runner, example_prompts, model: str, dtype: str
|
||||
) -> None:
|
||||
with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
|
||||
vllm_config = vllm_model.llm.llm_engine.vllm_config
|
||||
cache_config = vllm_config.cache_config
|
||||
assert not cache_config.enable_prefix_caching
|
||||
@@ -1,212 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from ....utils import RemoteOpenAIServer
|
||||
from .embed_utils import run_client_embeddings
|
||||
|
||||
MODEL_NAME = "BAAI/bge-m3"
|
||||
MAX_MODEL_LEN = 512
|
||||
|
||||
|
||||
# Example from https://huggingface.co/BAAI/bge-m3
|
||||
sentences_1 = ["What is BGE M3?", "Definition of BM25"]
|
||||
sentences_2 = [
|
||||
"BGE M3 is an embedding model supporting dense retrieval, "
|
||||
"lexical matching and multi-vector interaction.",
|
||||
"BM25 is a bag-of-words retrieval function that ranks a set "
|
||||
"of documents based on the query terms appearing in each document",
|
||||
]
|
||||
|
||||
similarity_reference = [[0.6259, 0.3474], [0.3309, 0.6734]]
|
||||
lexical_score_reference = [0.19554901123046875, 0.0]
|
||||
colbert_score_reference = [0.7797, 0.4620]
|
||||
SUPPORTED_TASKS = ["embed", "token_embed", "token_classify"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=SUPPORTED_TASKS)
|
||||
def pooling_task(request):
|
||||
yield request.param
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(pooling_task):
|
||||
args = [
|
||||
"--max-model-len",
|
||||
str(MAX_MODEL_LEN),
|
||||
"--hf-overrides",
|
||||
'{"architectures": ["BgeM3EmbeddingModel"]}',
|
||||
"--pooler-config.task",
|
||||
pooling_task,
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bge_m3_api_server_embedding(server, pooling_task):
|
||||
client = server.get_async_client()
|
||||
|
||||
if pooling_task != "embed":
|
||||
with pytest.raises(openai.InternalServerError):
|
||||
await run_client_embeddings(
|
||||
client,
|
||||
MODEL_NAME,
|
||||
sentences_1,
|
||||
)
|
||||
return
|
||||
|
||||
embeddings_list_1 = await run_client_embeddings(
|
||||
client,
|
||||
MODEL_NAME,
|
||||
sentences_1,
|
||||
)
|
||||
embeddings_list_2 = await run_client_embeddings(
|
||||
client,
|
||||
MODEL_NAME,
|
||||
sentences_2,
|
||||
)
|
||||
|
||||
embeddings_1 = torch.tensor(embeddings_list_1)
|
||||
embeddings_2 = torch.tensor(embeddings_list_2)
|
||||
similarity = embeddings_1 @ embeddings_2.T
|
||||
|
||||
# reference values from BAAI/bge-m3 documentation
|
||||
reference = torch.tensor(similarity_reference)
|
||||
|
||||
assert torch.allclose(similarity, reference, rtol=0.01)
|
||||
|
||||
|
||||
async def tokenize(client: openai.AsyncOpenAI, sentences: list[str]) -> list[list[int]]:
|
||||
futures = []
|
||||
for sentence in sentences:
|
||||
futures.append(
|
||||
client.post(
|
||||
"../tokenize",
|
||||
body={"model": MODEL_NAME, "prompt": sentence},
|
||||
cast_to=httpx.Response,
|
||||
)
|
||||
)
|
||||
return [(await future).json()["tokens"] for future in futures]
|
||||
|
||||
|
||||
async def sparse_embeddings(
|
||||
client: openai.AsyncOpenAI, sentences: list[str]
|
||||
) -> list[dict[int, float]]:
|
||||
all_tokens = await tokenize(client, sentences)
|
||||
result = await client.post(
|
||||
"../pooling",
|
||||
body={"model": MODEL_NAME, "input": sentences, "task": "token_classify"},
|
||||
cast_to=httpx.Response,
|
||||
)
|
||||
all_embeddings = [data["data"] for data in result.json()["data"]]
|
||||
|
||||
ret = []
|
||||
|
||||
for sent_tokens, sent_emb in zip(all_tokens, all_embeddings):
|
||||
token_embs = dict[int, float]()
|
||||
if sent_tokens[0] == 0:
|
||||
sent_tokens = sent_tokens[1:]
|
||||
for token, val in zip(sent_tokens, sent_emb):
|
||||
token_embs[token] = max(val, token_embs.get(token, 0.0))
|
||||
ret.append(token_embs)
|
||||
return ret
|
||||
|
||||
|
||||
# Based on https://github.com/FlagOpen/FlagEmbedding/blob/6fd176266f2382878bcc69cd656cff425d52f49b/FlagEmbedding/inference/embedder/encoder_only/m3.py#L129
|
||||
def compute_lexical_matching_score(
|
||||
lw1: dict[int, float], lw2: dict[int, float]
|
||||
) -> float:
|
||||
scores = 0.0
|
||||
for token, weight in lw1.items():
|
||||
if token in lw2:
|
||||
scores += weight * lw2[token]
|
||||
return scores
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bge_m3_api_server_sparse_embedding(server, pooling_task):
|
||||
client = server.get_async_client()
|
||||
|
||||
if pooling_task != "token_classify":
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
await sparse_embeddings(client, sentences_1)
|
||||
return
|
||||
|
||||
embeddings_1 = await sparse_embeddings(client, sentences_1)
|
||||
embeddings_2 = await sparse_embeddings(client, sentences_2)
|
||||
|
||||
lexical_scores_1_0_x_2_0 = compute_lexical_matching_score(
|
||||
embeddings_1[0], embeddings_2[0]
|
||||
)
|
||||
assert lexical_scores_1_0_x_2_0 == pytest.approx(
|
||||
lexical_score_reference[0], rel=0.01
|
||||
)
|
||||
|
||||
lexical_scores_1_0_x_1_1 = compute_lexical_matching_score(
|
||||
embeddings_1[0], embeddings_1[1]
|
||||
)
|
||||
assert lexical_scores_1_0_x_1_1 == pytest.approx(
|
||||
lexical_score_reference[1], rel=0.01
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bge_m3_api_server_sparse_embedding_corner_case(server, pooling_task):
|
||||
if pooling_task != "token_classify":
|
||||
return
|
||||
|
||||
client = server.get_async_client()
|
||||
embeddings = await sparse_embeddings(client, ["Hi"])
|
||||
assert len(embeddings) == 1
|
||||
assert 2673 in embeddings[0]
|
||||
assert embeddings[0][2673] == pytest.approx(0.26710861921310425, rel=0.01)
|
||||
|
||||
|
||||
# https://github.com/FlagOpen/FlagEmbedding/blob/6fd176266f2382878bcc69cd656cff425d52f49b/FlagEmbedding/inference/embedder/encoder_only/m3.py#L163
|
||||
def colbert_score(q_reps: torch.Tensor, p_reps: torch.Tensor) -> torch.Tensor:
|
||||
token_scores = torch.einsum("in,jn->ij", q_reps, p_reps)
|
||||
scores, _ = token_scores.max(-1)
|
||||
scores = torch.sum(scores) / q_reps.size(0)
|
||||
return scores
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bge_m3_api_server_multi_vector(server, pooling_task):
|
||||
client = server.get_async_client()
|
||||
|
||||
if pooling_task != "token_embed":
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
await client.post(
|
||||
"../pooling",
|
||||
body={"model": MODEL_NAME, "input": sentences_1, "task": "token_embed"},
|
||||
cast_to=httpx.Response,
|
||||
)
|
||||
return
|
||||
|
||||
result_1 = await client.post(
|
||||
"../pooling",
|
||||
body={"model": MODEL_NAME, "input": sentences_1, "task": "token_embed"},
|
||||
cast_to=httpx.Response,
|
||||
)
|
||||
embeddings_1 = [torch.tensor(data["data"]) for data in result_1.json()["data"]]
|
||||
|
||||
result_2 = await client.post(
|
||||
"../pooling",
|
||||
body={"model": MODEL_NAME, "input": sentences_2, "task": "token_embed"},
|
||||
cast_to=httpx.Response,
|
||||
)
|
||||
embeddings_2 = [torch.tensor(data["data"]) for data in result_2.json()["data"]]
|
||||
|
||||
colbert_score_1_0_x_2_0 = colbert_score(embeddings_1[0], embeddings_2[0])
|
||||
assert colbert_score_1_0_x_2_0 == pytest.approx(
|
||||
colbert_score_reference[0], rel=0.01
|
||||
)
|
||||
colbert_score_1_0_x_2_1 = colbert_score(embeddings_1[0], embeddings_2[1])
|
||||
assert colbert_score_1_0_x_2_1 == pytest.approx(
|
||||
colbert_score_reference[1], rel=0.01
|
||||
)
|
||||
@@ -1,108 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForSequenceClassification
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
pytest.param(
|
||||
"jason9693/Qwen2.5-1.5B-apeach",
|
||||
marks=[
|
||||
pytest.mark.core_model,
|
||||
pytest.mark.cpu_model,
|
||||
pytest.mark.slow_test,
|
||||
],
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"] if current_platform.is_rocm() else ["float"])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.classify(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.classify(example_prompts)
|
||||
|
||||
# check logits difference
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = torch.tensor(hf_output)
|
||||
vllm_output = torch.tensor(vllm_output)
|
||||
|
||||
# the tolerance value of 1e-2 is selected based on the
|
||||
# half datatype tests in
|
||||
# tests/models/language/pooling/test_embedding.py
|
||||
assert torch.allclose(
|
||||
hf_output,
|
||||
vllm_output,
|
||||
rtol=2e-3 if dtype == "float" else 1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
def test_bert_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
|
||||
model = "cross-encoder/ms-marco-TinyBERT-L-2-v2"
|
||||
score_inputs = (
|
||||
"What is the capital of France?",
|
||||
[
|
||||
"Paris.",
|
||||
"Paris is the capital and largest city of France.",
|
||||
"William Shakespeare wrote Hamlet in the early seventeenth century.",
|
||||
],
|
||||
)
|
||||
prompt_batches = [
|
||||
["short input"],
|
||||
[
|
||||
"short input",
|
||||
"a longer input that exercises mixed sequence lengths",
|
||||
],
|
||||
]
|
||||
|
||||
with hf_runner(
|
||||
model, dtype="half", auto_cls=AutoModelForSequenceClassification
|
||||
) as hf_model:
|
||||
# HfRunner uses problem_type to preserve the model's
|
||||
# sbert_ce_default_activation_function=Identity raw logits.
|
||||
hf_model.config.problem_type = "regression"
|
||||
hf_outputs = [hf_model.classify(prompts) for prompts in prompt_batches]
|
||||
|
||||
text_1, text_2 = score_inputs
|
||||
text_pairs = [[text_1, document] for document in text_2]
|
||||
with hf_runner(model, dtype="half", is_cross_encoder=True) as hf_model:
|
||||
hf_scores = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype="half",
|
||||
max_model_len=64,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = [vllm_model.classify(prompts) for prompts in prompt_batches]
|
||||
vllm_scores = vllm_model.score(*score_inputs)
|
||||
|
||||
for hf_batch, vllm_batch in zip(hf_outputs, vllm_outputs):
|
||||
hf_tensor = torch.tensor(hf_batch)
|
||||
vllm_tensor = torch.tensor(vllm_batch)
|
||||
assert vllm_tensor.shape == hf_tensor.shape
|
||||
assert torch.allclose(vllm_tensor, hf_tensor, rtol=1e-2, atol=1e-4)
|
||||
|
||||
assert torch.allclose(
|
||||
torch.tensor(vllm_scores),
|
||||
torch.tensor(hf_scores),
|
||||
rtol=1e-2,
|
||||
atol=1e-4,
|
||||
)
|
||||
@@ -1,374 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ColBERT late interaction scoring.
|
||||
|
||||
Tests are parametrized across multiple ColBERT backbones to ensure the
|
||||
generic ColBERT support works with different encoder architectures.
|
||||
"""
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Model definitions: (model_name, colbert_dim, extra vllm_runner kwargs)
|
||||
# -----------------------------------------------------------------------
|
||||
COLBERT_MODELS = {
|
||||
"bert": {
|
||||
"model": "answerdotai/answerai-colbert-small-v1",
|
||||
"colbert_dim": 96,
|
||||
"max_model_len": 512,
|
||||
"extra_kwargs": {},
|
||||
"hf_comparison": {
|
||||
"weights_file": "model.safetensors",
|
||||
"weights_key": "linear.weight",
|
||||
"trust_remote_code": False,
|
||||
"model_cls": "BertModel",
|
||||
},
|
||||
},
|
||||
"modernbert": {
|
||||
"model": "lightonai/GTE-ModernColBERT-v1",
|
||||
"colbert_dim": 128,
|
||||
"max_model_len": 299,
|
||||
"extra_kwargs": {
|
||||
"hf_overrides": {
|
||||
"architectures": ["ColBERTModernBertModel"],
|
||||
},
|
||||
},
|
||||
"hf_comparison": {
|
||||
"weights_file": "1_Dense/model.safetensors",
|
||||
"weights_key": "linear.weight",
|
||||
"trust_remote_code": False,
|
||||
"model_cls": "AutoModel",
|
||||
},
|
||||
},
|
||||
"jina": {
|
||||
"model": "jinaai/jina-colbert-v2",
|
||||
"colbert_dim": 128,
|
||||
"max_model_len": 8192,
|
||||
"extra_kwargs": {
|
||||
"hf_overrides": {
|
||||
"architectures": ["ColBERTJinaRobertaModel"],
|
||||
},
|
||||
},
|
||||
"hf_comparison": {
|
||||
"weights_file": "model.safetensors",
|
||||
"weights_key": "linear.weight",
|
||||
"trust_remote_code": True,
|
||||
"model_cls": "AutoModel",
|
||||
},
|
||||
},
|
||||
"lfm2": {
|
||||
"model": "LiquidAI/LFM2-ColBERT-350M",
|
||||
"colbert_dim": 128,
|
||||
"max_model_len": 511,
|
||||
"extra_kwargs": {
|
||||
"hf_overrides": {
|
||||
"architectures": ["ColBERTLfm2Model"],
|
||||
},
|
||||
},
|
||||
"hf_comparison": {
|
||||
"weights_file": "1_Dense/model.safetensors",
|
||||
"weights_key": "linear.weight",
|
||||
"trust_remote_code": False,
|
||||
"model_cls": "AutoModel",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
TEXTS_1 = [
|
||||
"What is the capital of France?",
|
||||
"What is the capital of Germany?",
|
||||
]
|
||||
|
||||
TEXTS_2 = [
|
||||
"The capital of France is Paris.",
|
||||
"The capital of Germany is Berlin.",
|
||||
]
|
||||
|
||||
DTYPE = "half"
|
||||
|
||||
|
||||
def _load_hf_model(model_name: str, hf_spec: dict, device: torch.device):
|
||||
"""Load HF model on the given device with a compatible attention impl."""
|
||||
from transformers import AutoModel, BertModel
|
||||
|
||||
cls = BertModel if hf_spec["model_cls"] == "BertModel" else AutoModel
|
||||
trust = hf_spec.get("trust_remote_code", False)
|
||||
|
||||
# Flash / Triton kernels require GPU tensors; fall back to eager on CPU.
|
||||
extra = {}
|
||||
if device.type == "cpu":
|
||||
extra["attn_implementation"] = "eager"
|
||||
|
||||
model = cls.from_pretrained(
|
||||
model_name,
|
||||
trust_remote_code=trust,
|
||||
**extra,
|
||||
).to(device)
|
||||
model.eval()
|
||||
|
||||
# Transformers 5.0 weight materialization can clear non-persistent
|
||||
# buffers (e.g. rotary inv_freq) that were registered with
|
||||
# persistent=False. Re-compute them so the model produces valid output.
|
||||
for mod in model.modules():
|
||||
if hasattr(mod, "_compute_inv_freq") and hasattr(mod, "inv_freq"):
|
||||
mod.inv_freq = mod._compute_inv_freq(device=device)
|
||||
|
||||
return model
|
||||
|
||||
|
||||
def _load_projection_weight(model_name: str, hf_spec: dict, device: torch.device):
|
||||
"""Download and return the ColBERT linear projection weight."""
|
||||
from safetensors.torch import load_file
|
||||
|
||||
from vllm.transformers_utils.repo_utils import hf_api, with_retry
|
||||
|
||||
path = with_retry(
|
||||
lambda: hf_api().hf_hub_download(model_name, filename=hf_spec["weights_file"]),
|
||||
f"Error downloading {hf_spec['weights_file']} from {model_name}",
|
||||
)
|
||||
weights = load_file(path)
|
||||
return weights[hf_spec["weights_key"]].to(device)
|
||||
|
||||
|
||||
def _compute_hf_colbert_embeddings(model, tokenizer, linear_weight, texts, device):
|
||||
"""Run HF model + projection and return L2-normalised token embeddings."""
|
||||
import torch.nn.functional as F
|
||||
|
||||
embeddings = []
|
||||
for text in texts:
|
||||
inputs = tokenizer(text, return_tensors="pt").to(device)
|
||||
with torch.no_grad():
|
||||
hidden = model(**inputs).last_hidden_state.float()
|
||||
projected = F.linear(hidden, linear_weight.float())
|
||||
normalised = F.normalize(projected, p=2, dim=-1)
|
||||
embeddings.append(normalised.squeeze(0).cpu())
|
||||
return embeddings
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _hf_colbert_model(model_name: str, hf_spec: dict, device: torch.device):
|
||||
"""Load the HF backbone + ColBERT projection, freeing the GPU on exit.
|
||||
|
||||
These live outside any runner context, so without explicit cleanup ROCm
|
||||
keeps the VRAM resident and the next backend parametrization (or test)
|
||||
OOMs on startup.
|
||||
"""
|
||||
hf_model = _load_hf_model(model_name, hf_spec, device)
|
||||
linear_weight = _load_projection_weight(model_name, hf_spec, device)
|
||||
try:
|
||||
yield hf_model, linear_weight
|
||||
finally:
|
||||
del hf_model, linear_weight
|
||||
cleanup_dist_env_and_memory()
|
||||
wait_for_memory_to_settle()
|
||||
|
||||
|
||||
def _assert_embeddings_close(vllm_outputs, hf_embeddings):
|
||||
"""Assert that vLLM and HuggingFace embeddings match."""
|
||||
for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)):
|
||||
vllm_emb = torch.as_tensor(vllm_out).float()
|
||||
|
||||
assert hf_emb.shape == vllm_emb.shape, (
|
||||
f"Shape mismatch for text {i}: HF {hf_emb.shape} vs vLLM {vllm_emb.shape}"
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
vllm_emb,
|
||||
hf_emb,
|
||||
rtol=1e-2,
|
||||
atol=1e-2,
|
||||
msg=f"Embedding mismatch for text {i}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(params=list(COLBERT_MODELS.keys()), scope="class")
|
||||
def colbert_spec(request):
|
||||
"""Return the model spec dict for the current parametrization."""
|
||||
return COLBERT_MODELS[request.param]
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def colbert_model_name(colbert_spec):
|
||||
return colbert_spec["model"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def colbert_dim(colbert_spec):
|
||||
return colbert_spec["colbert_dim"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def colbert_max_model_len(colbert_spec):
|
||||
return colbert_spec["max_model_len"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def colbert_extra_kwargs(colbert_spec):
|
||||
return colbert_spec["extra_kwargs"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def colbert_model(
|
||||
vllm_runner,
|
||||
colbert_model_name,
|
||||
colbert_max_model_len,
|
||||
colbert_extra_kwargs,
|
||||
):
|
||||
with vllm_runner(
|
||||
colbert_model_name,
|
||||
runner="pooling",
|
||||
dtype=DTYPE,
|
||||
max_model_len=colbert_max_model_len,
|
||||
enforce_eager=True,
|
||||
**colbert_extra_kwargs,
|
||||
) as vllm_model:
|
||||
yield vllm_model
|
||||
|
||||
|
||||
class TestColbertSharedEngine:
|
||||
"""Tests sharing one engine per model.
|
||||
|
||||
Class-scoped so the engine is released before `test_colbert_hf_comparison`,
|
||||
which needs a runner of its own and cannot start while this one holds VRAM.
|
||||
"""
|
||||
|
||||
def test_colbert_token_embed(self, colbert_model, colbert_dim):
|
||||
"""Test that ColBERT model produces token embeddings."""
|
||||
outputs = colbert_model.token_embed([TEXTS_1[0]])
|
||||
|
||||
assert len(outputs) == 1
|
||||
emb = torch.as_tensor(outputs[0])
|
||||
assert emb.dim() == 2
|
||||
assert emb.shape[1] == colbert_dim
|
||||
assert emb.shape[0] > 1
|
||||
|
||||
def test_colbert_late_interaction_1_to_1(self, colbert_model):
|
||||
"""Test ColBERT late interaction scoring with 1:1 query-document pair."""
|
||||
q_outputs = colbert_model.token_embed([TEXTS_1[0]])
|
||||
d_outputs = colbert_model.token_embed([TEXTS_2[0]])
|
||||
|
||||
q_emb = torch.as_tensor(q_outputs[0])
|
||||
d_emb = torch.as_tensor(d_outputs[0])
|
||||
|
||||
manual_score = compute_maxsim_score(q_emb, d_emb).item()
|
||||
|
||||
vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2[0])
|
||||
|
||||
assert len(vllm_scores) == 1
|
||||
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
|
||||
|
||||
def test_colbert_late_interaction_1_to_N(self, colbert_model):
|
||||
"""Test ColBERT late interaction scoring with 1:N query-documents."""
|
||||
q_outputs = colbert_model.token_embed([TEXTS_1[0]])
|
||||
d_outputs = colbert_model.token_embed(TEXTS_2)
|
||||
|
||||
q_emb = torch.as_tensor(q_outputs[0])
|
||||
|
||||
manual_scores = []
|
||||
for d_out in d_outputs:
|
||||
d_emb = torch.as_tensor(d_out)
|
||||
manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
|
||||
|
||||
vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2)
|
||||
|
||||
assert len(vllm_scores) == 2
|
||||
for i in range(2):
|
||||
assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
|
||||
|
||||
def test_colbert_late_interaction_N_to_N(self, colbert_model):
|
||||
"""Test ColBERT late interaction scoring with N:N query-documents."""
|
||||
q_outputs = colbert_model.token_embed(TEXTS_1)
|
||||
d_outputs = colbert_model.token_embed(TEXTS_2)
|
||||
|
||||
manual_scores = []
|
||||
for q_out, d_out in zip(q_outputs, d_outputs):
|
||||
q_emb = torch.as_tensor(q_out)
|
||||
d_emb = torch.as_tensor(d_out)
|
||||
manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
|
||||
|
||||
vllm_scores = colbert_model.score(TEXTS_1, TEXTS_2)
|
||||
|
||||
assert len(vllm_scores) == 2
|
||||
for i in range(2):
|
||||
assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
|
||||
|
||||
def test_colbert_relevance_ordering(self, colbert_model):
|
||||
"""Test that ColBERT scores relevant documents higher than irrelevant."""
|
||||
query = "What is machine learning?"
|
||||
documents = [
|
||||
"Machine learning is a subset of artificial intelligence.",
|
||||
"Python is a programming language.",
|
||||
"Deep learning uses neural networks.",
|
||||
]
|
||||
|
||||
scores = colbert_model.score(query, documents)
|
||||
|
||||
assert len(scores) == 3
|
||||
assert scores[0] > scores[1], "ML doc should score higher than Python doc"
|
||||
assert scores[2] > scores[1], "DL doc should score higher than Python doc"
|
||||
|
||||
def test_colbert_embed_not_supported(self, colbert_model):
|
||||
"""Test that ColBERT model does not support the embed task."""
|
||||
with pytest.raises(ValueError, match="Embedding API is not supported"):
|
||||
colbert_model.embed([TEXTS_1[0]])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "use_v2"),
|
||||
[
|
||||
pytest.param("bert", True, id="bert-v2"),
|
||||
pytest.param("modernbert", True, id="modernbert-v2"),
|
||||
pytest.param("jina", True, id="jina-v2"),
|
||||
pytest.param("lfm2", True, id="lfm2-v2"),
|
||||
],
|
||||
)
|
||||
def test_colbert_hf_comparison(vllm_runner, monkeypatch, backend, use_v2):
|
||||
"""Test that vLLM ColBERT embeddings match HuggingFace for each backend."""
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
spec = COLBERT_MODELS[backend]
|
||||
hf_spec = spec["hf_comparison"]
|
||||
extra_kwargs = spec["extra_kwargs"]
|
||||
model_name = spec["model"]
|
||||
assert isinstance(model_name, str)
|
||||
assert isinstance(hf_spec, dict)
|
||||
assert isinstance(extra_kwargs, dict)
|
||||
test_texts = [TEXTS_1[0], TEXTS_2[0]]
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2 else "0")
|
||||
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
runner="pooling",
|
||||
dtype="float32",
|
||||
max_model_len=spec["max_model_len"],
|
||||
enforce_eager=True,
|
||||
**extra_kwargs,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner == use_v2
|
||||
vllm_outputs = vllm_model.token_embed(test_texts)
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
hf_tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_name,
|
||||
trust_remote_code=hf_spec.get("trust_remote_code", False),
|
||||
)
|
||||
with _hf_colbert_model(model_name, hf_spec, device) as (hf_model, linear_weight):
|
||||
hf_embeddings = _compute_hf_colbert_embeddings(
|
||||
hf_model,
|
||||
hf_tokenizer,
|
||||
linear_weight,
|
||||
test_texts,
|
||||
device,
|
||||
)
|
||||
|
||||
_assert_embeddings_close(vllm_outputs, hf_embeddings)
|
||||
@@ -1,219 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import PoolerConfig
|
||||
|
||||
from ...utils import check_embeddings_close
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
# Be careful of the order of models, decoder-only models should be
|
||||
# placed before encoder-only models, otherwise `Qwen2.5-0.5B-Instruct`
|
||||
# case won't pass because gte-Qwen2-1.5B-instruct will cache custom
|
||||
# model code with bidirectional attention.
|
||||
# [Decoder-only]
|
||||
pytest.param(
|
||||
"BAAI/bge-multilingual-gemma2",
|
||||
marks=[pytest.mark.core_model, pytest.mark.slow_test],
|
||||
),
|
||||
pytest.param(
|
||||
"intfloat/e5-mistral-7b-instruct",
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
pytest.param(
|
||||
"ssmits/Qwen2-7B-Instruct-embed-base", marks=[pytest.mark.cpu_model]
|
||||
),
|
||||
# [Encoder-only]
|
||||
pytest.param(
|
||||
"BAAI/bge-base-en-v1.5",
|
||||
marks=[
|
||||
pytest.mark.core_model,
|
||||
pytest.mark.cpu_model,
|
||||
pytest.mark.slow_test,
|
||||
],
|
||||
),
|
||||
pytest.param("sentence-transformers/all-MiniLM-L12-v2"),
|
||||
pytest.param("intfloat/multilingual-e5-small"),
|
||||
# [Cross-Encoder]
|
||||
pytest.param(
|
||||
"sentence-transformers/stsb-roberta-base-v2",
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model,
|
||||
) -> None:
|
||||
vllm_extra_kwargs = {}
|
||||
if model == "ssmits/Qwen2-7B-Instruct-embed-base":
|
||||
vllm_extra_kwargs["pooler_config"] = PoolerConfig(
|
||||
seq_pooling_type="MEAN", use_activation=False
|
||||
)
|
||||
|
||||
max_model_len: int | None = 512
|
||||
if model in [
|
||||
"sentence-transformers/all-MiniLM-L12-v2",
|
||||
"sentence-transformers/stsb-roberta-base-v2",
|
||||
]:
|
||||
max_model_len = None
|
||||
|
||||
# The example_prompts has ending "\n", for example:
|
||||
# "Write a short story about a robot that dreams for the first time.\n"
|
||||
# sentence_transformers will strip the input texts, see:
|
||||
# https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
|
||||
# This makes the input_ids different between hf_model and vllm_model.
|
||||
# So we need to strip the input texts to avoid test failing.
|
||||
example_prompts = [str(s).strip() for s in example_prompts]
|
||||
|
||||
with hf_runner(model, is_sentence_transformer=True) as hf_model:
|
||||
hf_outputs = hf_model.encode(example_prompts)
|
||||
|
||||
with vllm_runner(
|
||||
model, runner="pooling", max_model_len=max_model_len, **vllm_extra_kwargs
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.embed(example_prompts)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"BAAI/bge-base-en-v1.5",
|
||||
"intfloat/multilingual-e5-small",
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_encoder_only_model_runner_v2_attention(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
) -> None:
|
||||
prompts = [
|
||||
"short input",
|
||||
"a longer input that exercises mixed sequence lengths",
|
||||
]
|
||||
|
||||
with hf_runner(model, dtype="float", auto_cls=AutoModel) as hf_model:
|
||||
hf_outputs = []
|
||||
for prompt in prompts:
|
||||
inputs = hf_model.tokenizer(prompt, return_tensors="pt")
|
||||
output = hf_model.model(**hf_model.wrap_device(inputs))
|
||||
embedding = torch.nn.functional.normalize(
|
||||
output.last_hidden_state[0, -1].float(), dim=0
|
||||
)
|
||||
hf_outputs.append(embedding.cpu().tolist())
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype="float",
|
||||
max_model_len=64,
|
||||
max_num_seqs=2,
|
||||
gpu_memory_utilization=0.25,
|
||||
pooler_config=PoolerConfig(
|
||||
task="embed", seq_pooling_type="LAST", use_activation=True
|
||||
),
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.embed(prompts)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
def test_encoder_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
|
||||
model = "sentence-transformers/all-MiniLM-L6-v2"
|
||||
prompt_batches = [
|
||||
["short input"],
|
||||
[
|
||||
"short input",
|
||||
"a longer input that exercises mixed sequence lengths",
|
||||
],
|
||||
]
|
||||
|
||||
with hf_runner(model, is_sentence_transformer=True) as hf_model:
|
||||
hf_outputs = [hf_model.encode(prompts) for prompts in prompt_batches]
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
max_model_len=64,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = [vllm_model.embed(prompts) for prompts in prompt_batches]
|
||||
|
||||
for hf_batch, vllm_batch in zip(hf_outputs, vllm_outputs):
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_batch,
|
||||
embeddings_1_lst=vllm_batch,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
def test_matryoshka_dimensions_model_runner_v2(
|
||||
hf_runner, vllm_runner, monkeypatch
|
||||
) -> None:
|
||||
model = "Snowflake/snowflake-arctic-embed-m-v1.5"
|
||||
prompts = ["short input", "a longer input for a different output width"]
|
||||
dimensions = [None, 256]
|
||||
|
||||
with hf_runner(model, is_sentence_transformer=True) as hf_model:
|
||||
hf_outputs = hf_model.encode(prompts)
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
max_model_len=64,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.embed(
|
||||
prompts,
|
||||
pooling_params=[PoolingParams(dimensions=d) for d in dimensions],
|
||||
)
|
||||
|
||||
expected_outputs = []
|
||||
for output, dimension in zip(hf_outputs, dimensions):
|
||||
output = torch.as_tensor(output)
|
||||
if dimension is not None:
|
||||
output = torch.nn.functional.normalize(output[:dimension], dim=0)
|
||||
expected_outputs.append(output.tolist())
|
||||
|
||||
assert [len(output) for output in vllm_outputs] == [768, 256]
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=expected_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
@@ -1,48 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import TokensPrompt
|
||||
from vllm.config import PoolerConfig
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["Qwen/Qwen3-0.6B"],
|
||||
)
|
||||
@torch.inference_mode
|
||||
def test_extract_hidden_states(hf_runner, vllm_runner, model: str):
|
||||
n_prompt_tokens = [55, 56, 57]
|
||||
token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=128,
|
||||
enforce_eager=True,
|
||||
runner="pooling",
|
||||
pooler_config=PoolerConfig(task="token_embed"),
|
||||
enable_prefix_caching=True,
|
||||
) as vllm_model:
|
||||
pooling_outputs = vllm_model.llm.encode(
|
||||
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
|
||||
pooling_task="token_embed",
|
||||
)
|
||||
|
||||
for n, output in zip(n_prompt_tokens, pooling_outputs):
|
||||
assert len(output.prompt_token_ids) == n
|
||||
assert len(output.outputs.data) == n
|
||||
assert output.num_cached_tokens == 0
|
||||
|
||||
# test enable_prefix_caching plus all pooling
|
||||
# we need to skip reading cache at this request by
|
||||
# request.skip_reading_prefix_cache
|
||||
pooling_outputs = vllm_model.llm.encode(
|
||||
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
|
||||
pooling_task="token_embed",
|
||||
)
|
||||
|
||||
for n, output in zip(n_prompt_tokens, pooling_outputs):
|
||||
assert len(output.prompt_token_ids) == n
|
||||
assert len(output.outputs.data) == n
|
||||
assert output.num_cached_tokens == 0
|
||||
@@ -1,47 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForSequenceClassification
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["nie3e/sentiment-polish-gpt2-small"],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_classify_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with hf_runner(
|
||||
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.classify(example_prompts)
|
||||
|
||||
for head_dtype_str in ["float32", "model"]:
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
hf_overrides={"head_dtype": head_dtype_str},
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
model_dtype = model_config.dtype
|
||||
head_dtype = model_config.head_dtype
|
||||
|
||||
if head_dtype_str == "float32":
|
||||
assert head_dtype == torch.float32
|
||||
elif head_dtype_str == "model":
|
||||
assert head_dtype == model_dtype
|
||||
|
||||
vllm_outputs = vllm_model.classify(example_prompts)
|
||||
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = torch.tensor(hf_output).float()
|
||||
vllm_output = torch.tensor(vllm_output).float()
|
||||
|
||||
assert torch.allclose(hf_output, vllm_output, atol=1e-2)
|
||||
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Backbone validation for Jina Embeddings V5.
|
||||
|
||||
The V5 family ships two backbones under one `architectures` entry: `-small` is
|
||||
a Qwen3 decoder, while `-nano` is a bidirectional EuroBERT encoder. Upstream
|
||||
ships a separate `configuration_*.py` per repository, so the only signal
|
||||
distinguishing them is `is_decoder`, which the encoder variant sets to False.
|
||||
`JinaEmbeddingsV5ModelConfig` uses it to enable bidirectional attention for the
|
||||
encoder variant; `JinaEmbeddingsV5Model` then dispatches to the correct backbone.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.config import (
|
||||
MODELS_CONFIG_MAP,
|
||||
JinaEmbeddingsV5ModelConfig,
|
||||
)
|
||||
|
||||
|
||||
def _model_config(hf_config: PreTrainedConfig) -> ModelConfig:
|
||||
"""Minimal stand-in for ModelConfig; only hf_config is read."""
|
||||
return cast(ModelConfig, SimpleNamespace(hf_config=hf_config))
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_registered_for_the_architecture():
|
||||
"""The handler only runs if it is wired to the architecture name."""
|
||||
assert MODELS_CONFIG_MAP["JinaEmbeddingsV5Model"] is JinaEmbeddingsV5ModelConfig
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_encoder_backbone_enables_bidirectional_attention():
|
||||
"""An encoder checkpoint (is_decoder=False) is supported.
|
||||
|
||||
The handler sets is_causal=False so the Llama backbone uses
|
||||
EncoderOnlyAttention; JinaEmbeddingsV5Model then dispatches to the encoder
|
||||
implementation.
|
||||
"""
|
||||
hf_config = PreTrainedConfig(is_decoder=False)
|
||||
|
||||
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(_model_config(hf_config))
|
||||
|
||||
assert hf_config.is_causal is False
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_supported_decoder_backbone_is_accepted():
|
||||
"""The Qwen3-based variants must keep loading.
|
||||
|
||||
`-small` omits `is_decoder` entirely, so an absent attribute has to be
|
||||
treated as a decoder. The first assertion pins that assumption: if
|
||||
PreTrainedConfig ever gains an `is_decoder=False` default, this fails here
|
||||
rather than silently rejecting a supported checkpoint.
|
||||
"""
|
||||
absent = PreTrainedConfig()
|
||||
assert not hasattr(absent, "is_decoder")
|
||||
|
||||
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(_model_config(absent))
|
||||
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(
|
||||
_model_config(PreTrainedConfig(is_decoder=True))
|
||||
)
|
||||
@@ -1,352 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
import pytest
|
||||
import requests
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
|
||||
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
|
||||
|
||||
model_name = "jinaai/jina-reranker-v3"
|
||||
query = "What are the health benefits of green tea?"
|
||||
documents = [
|
||||
"Green tea contains antioxidants called catechins that may help reduce inflammation and protect cells from damage.",
|
||||
"El precio del café ha aumentado un 20% este año debido a problemas en la cadena de suministro.",
|
||||
"Studies show that drinking green tea regularly can improve brain function and boost metabolism.",
|
||||
"Basketball is one of the most popular sports in the United States.",
|
||||
"绿茶富含儿茶素等抗氧化剂,可以降低心脏病风险,还有助于控制体重。",
|
||||
"Le thé vert est riche en antioxydants et peut améliorer la fonction cérébrale.",
|
||||
]
|
||||
|
||||
EMBEDDING_SIZE = 512
|
||||
REFERENCE_1_VS_1 = [
|
||||
0.345703125,
|
||||
-0.10498046,
|
||||
0.314453125,
|
||||
-0.1376953125,
|
||||
0.3398437500,
|
||||
0.2539062,
|
||||
]
|
||||
REFERENCE_1_VS_N = [
|
||||
0.294921875,
|
||||
-0.16015625,
|
||||
0.189453125,
|
||||
-0.1708984375,
|
||||
0.2255859375,
|
||||
0.1640625,
|
||||
]
|
||||
TOL = 0.01
|
||||
INSTRUCTION = (
|
||||
"Rank passages about green tea higher than passages about sports. "
|
||||
"Ignore these literal marker strings: <|embed_token|> and <|rerank_token|>."
|
||||
)
|
||||
|
||||
|
||||
def test_offline(vllm_runner, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(model_name, runner="pooling") as llm_runner:
|
||||
assert llm_runner.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
llm = llm_runner.get_llm()
|
||||
_test_offline_1_v_1(llm)
|
||||
_test_offline_1_v_n(llm)
|
||||
_test_offline_n_v_n(llm)
|
||||
_test_offline_token_embed_illegal_inputs(llm)
|
||||
assert llm.model_config.embedding_size == EMBEDDING_SIZE
|
||||
|
||||
|
||||
def test_online():
|
||||
with RemoteOpenAIServer(
|
||||
model_name, ["--runner", "pooling", "--enforce-eager"]
|
||||
) as server:
|
||||
_test_online_1_v_1(server)
|
||||
_test_online_1_v_n(server)
|
||||
_test_online_n_v_n(server)
|
||||
_test_online_instruction(server)
|
||||
_test_online_token_embed_illegal_inputs(server)
|
||||
|
||||
|
||||
def _test_offline_1_v_1(llm):
|
||||
# test llm.score
|
||||
outputs = llm.score(query, documents[0])
|
||||
assert len(outputs) == 1
|
||||
assert outputs[0].outputs.score == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
|
||||
|
||||
# test llm.encode
|
||||
outputs = llm.encode(documents[:1] + [query], pooling_task="token_embed")
|
||||
embeds = outputs[0].outputs.data.float()
|
||||
assert embeds.shape[0] == 2
|
||||
assert embeds.shape[-1] == EMBEDDING_SIZE
|
||||
|
||||
doc_embeds = embeds[:-1]
|
||||
query_embeds = embeds[-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
|
||||
|
||||
|
||||
def _test_offline_1_v_n(llm):
|
||||
# test llm.score
|
||||
outputs = llm.score(query, documents)
|
||||
assert len(outputs) == len(documents)
|
||||
|
||||
for expected, output in zip(REFERENCE_1_VS_N, outputs):
|
||||
actual = output.outputs.score
|
||||
assert actual == pytest.approx(expected, abs=TOL)
|
||||
|
||||
# test llm.encode
|
||||
outputs = llm.encode(documents + [query], pooling_task="token_embed")
|
||||
embeds = outputs[0].outputs.data.float()
|
||||
assert embeds.shape[0] == len(documents) + 1
|
||||
|
||||
doc_embeds = embeds[:-1]
|
||||
query_embeds = embeds[-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
|
||||
assert len(scores) == len(documents)
|
||||
for expected, actual in zip(REFERENCE_1_VS_N, scores):
|
||||
assert actual == pytest.approx(expected, abs=TOL)
|
||||
|
||||
|
||||
def _test_offline_n_v_n(llm):
|
||||
# test llm.score
|
||||
outputs = llm.score([query] * len(documents), documents)
|
||||
assert len(outputs) == len(documents)
|
||||
|
||||
for expected, output in zip(REFERENCE_1_VS_1, outputs):
|
||||
actual = output.outputs.score
|
||||
assert actual == pytest.approx(expected, abs=TOL)
|
||||
|
||||
# test llm.encode
|
||||
for doc, expected in zip(documents, REFERENCE_1_VS_1):
|
||||
outputs = llm.encode([doc, query], pooling_task="token_embed")
|
||||
embeds = outputs[0].outputs.data.float()
|
||||
assert embeds.shape[0] == 2
|
||||
|
||||
doc_embeds = embeds[:-1]
|
||||
query_embeds = embeds[-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
assert scores[0] == pytest.approx(expected, abs=TOL)
|
||||
|
||||
|
||||
def _test_offline_token_embed_illegal_inputs(llm):
|
||||
with pytest.raises(
|
||||
ValueError, match="The JinaForRanking model requires at least 2 inputs."
|
||||
):
|
||||
llm.encode([query], pooling_task="token_embed")
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="The JinaForRanking model only supports text as input."
|
||||
):
|
||||
llm.encode([1, 2, 3], pooling_task="token_embed")
|
||||
|
||||
|
||||
def _get_score_response(server, query, document, **extra_body):
|
||||
payload = {
|
||||
"model": model_name,
|
||||
"queries": query,
|
||||
"documents": document,
|
||||
}
|
||||
payload.update(extra_body)
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json=payload,
|
||||
)
|
||||
|
||||
score_response.raise_for_status()
|
||||
return ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
|
||||
def _get_scores(server, query, document):
|
||||
score = _get_score_response(server, query, document)
|
||||
|
||||
return [d.score for d in score.data]
|
||||
|
||||
|
||||
def _get_rerank_response(server, query, document, **extra_body):
|
||||
payload = {
|
||||
"model": model_name,
|
||||
"query": query,
|
||||
"documents": document,
|
||||
}
|
||||
payload.update(extra_body)
|
||||
rerank_response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json=payload,
|
||||
)
|
||||
|
||||
rerank_response.raise_for_status()
|
||||
return RerankResponse.model_validate(rerank_response.json())
|
||||
|
||||
|
||||
def _get_embeds(server, prompts: list[str]):
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"task": "token_embed",
|
||||
"input": prompts,
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
|
||||
return torch.as_tensor([d.data for d in poolings.data][0]).float()
|
||||
|
||||
|
||||
def _test_online_1_v_1(server):
|
||||
# test scoring api
|
||||
scores = _get_scores(server, query, documents[0])
|
||||
assert len(scores) == 1
|
||||
assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
|
||||
|
||||
# test pooling api
|
||||
embeds = _get_embeds(server, [documents[0], query])
|
||||
assert embeds.shape[0] == 2
|
||||
assert embeds.shape[-1] == EMBEDDING_SIZE
|
||||
|
||||
doc_embeds = embeds[:-1]
|
||||
query_embeds = embeds[-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
|
||||
|
||||
|
||||
def _test_online_1_v_n(server):
|
||||
# test scoring api
|
||||
scores = _get_scores(server, query, documents)
|
||||
assert len(scores) == len(documents)
|
||||
|
||||
for expected, actual in zip(REFERENCE_1_VS_N, scores):
|
||||
assert actual == pytest.approx(expected, abs=TOL)
|
||||
|
||||
# test pooling api
|
||||
embeds = _get_embeds(server, documents + [query])
|
||||
assert embeds.shape[0] == len(documents) + 1
|
||||
|
||||
doc_embeds = embeds[:-1]
|
||||
query_embeds = embeds[-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
|
||||
assert len(scores) == len(documents)
|
||||
for expected, actual in zip(REFERENCE_1_VS_N, scores):
|
||||
assert actual == pytest.approx(expected, abs=TOL)
|
||||
|
||||
|
||||
def _test_online_n_v_n(server):
|
||||
# test scoring api
|
||||
scores = _get_scores(server, [query] * len(documents), documents)
|
||||
assert len(scores) == len(documents)
|
||||
|
||||
for expected, actual in zip(REFERENCE_1_VS_1, scores):
|
||||
assert actual == pytest.approx(expected, abs=TOL)
|
||||
|
||||
# test pooling api
|
||||
for doc, expected in zip(documents, REFERENCE_1_VS_1):
|
||||
embeds = _get_embeds(server, [doc, query])
|
||||
assert embeds.shape[0] == 2
|
||||
|
||||
doc_embeds = embeds[:-1]
|
||||
query_embeds = embeds[-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
assert len(scores) == 1
|
||||
assert scores[0] == pytest.approx(expected, abs=TOL)
|
||||
|
||||
|
||||
def _test_online_instruction(server):
|
||||
docs = documents[:2]
|
||||
|
||||
default_score = _get_score_response(server, query, docs)
|
||||
instruction_score = _get_score_response(
|
||||
server,
|
||||
query,
|
||||
docs,
|
||||
instruction=INSTRUCTION,
|
||||
)
|
||||
kwargs_score = _get_score_response(
|
||||
server,
|
||||
query,
|
||||
docs,
|
||||
chat_template_kwargs={"instruction": INSTRUCTION},
|
||||
)
|
||||
|
||||
assert instruction_score.usage.prompt_tokens > default_score.usage.prompt_tokens
|
||||
assert kwargs_score.usage.prompt_tokens == instruction_score.usage.prompt_tokens
|
||||
assert len(instruction_score.data) == len(default_score.data)
|
||||
assert [d.score for d in kwargs_score.data] == pytest.approx(
|
||||
[d.score for d in instruction_score.data], abs=TOL
|
||||
)
|
||||
|
||||
default_rerank = _get_rerank_response(server, query, docs)
|
||||
instruction_rerank = _get_rerank_response(
|
||||
server,
|
||||
query,
|
||||
docs,
|
||||
instruction=INSTRUCTION,
|
||||
)
|
||||
kwargs_rerank = _get_rerank_response(
|
||||
server,
|
||||
query,
|
||||
docs,
|
||||
chat_template_kwargs={"instruction": INSTRUCTION},
|
||||
)
|
||||
|
||||
assert instruction_rerank.usage.prompt_tokens > default_rerank.usage.prompt_tokens
|
||||
assert kwargs_rerank.usage.prompt_tokens == instruction_rerank.usage.prompt_tokens
|
||||
assert len(instruction_rerank.results) == len(default_rerank.results)
|
||||
assert [r.relevance_score for r in kwargs_rerank.results] == pytest.approx(
|
||||
[r.relevance_score for r in instruction_rerank.results], abs=TOL
|
||||
)
|
||||
|
||||
|
||||
def _test_online_token_embed_illegal_inputs(server):
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"task": "token_embed",
|
||||
"input": [query],
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
assert response.json()["error"]["message"].startswith(
|
||||
"The JinaForRanking model requires at least 2 inputs."
|
||||
)
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"task": "token_embed",
|
||||
"input": [1, 2, 3],
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
assert response.json()["error"]["message"].startswith(
|
||||
"The JinaForRanking model only supports text as input."
|
||||
)
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"task": "token_embed",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
}
|
||||
],
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
assert response.json()["error"]["message"].startswith(
|
||||
"The JinaForRanking does not support chat Request."
|
||||
)
|
||||
@@ -1,227 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for max_tokens_per_doc and max_tokens_per_query."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import VLLM_PATH, RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse
|
||||
|
||||
TEMPLATE_DIR = str(VLLM_PATH / "examples/pooling/score/template")
|
||||
ExpectedPromptTokens = int | tuple[int, ...]
|
||||
|
||||
long_query = "What is the capital of France?" * 20
|
||||
long_doc = "The capital of France is Paris. " * 20
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestConfig:
|
||||
model: str
|
||||
args: list[str]
|
||||
without_truncated_prompt_tokens: ExpectedPromptTokens
|
||||
with_max_tokens_per_query_prompt_tokens: ExpectedPromptTokens
|
||||
with_max_tokens_per_doc_prompt_tokens: ExpectedPromptTokens
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens: ExpectedPromptTokens
|
||||
|
||||
|
||||
RERANK_CONFIGS = [
|
||||
# 1. cross-encoder
|
||||
TestConfig(
|
||||
model="jinaai/jina-reranker-v2-base-multilingual",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
without_truncated_prompt_tokens=284,
|
||||
with_max_tokens_per_query_prompt_tokens=154,
|
||||
with_max_tokens_per_doc_prompt_tokens=154,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=24,
|
||||
),
|
||||
# 2. cross-encoder + score template
|
||||
TestConfig(
|
||||
model="Qwen/Qwen3-Reranker-0.6B",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--hf-overrides",
|
||||
json.dumps(
|
||||
{
|
||||
"architectures": ["Qwen3ForSequenceClassification"],
|
||||
"classifier_from_token": ["no", "yes"],
|
||||
"is_original_qwen3_reranker": True,
|
||||
}
|
||||
),
|
||||
"--chat-template",
|
||||
os.path.join(TEMPLATE_DIR, "qwen3_reranker.jinja"),
|
||||
],
|
||||
without_truncated_prompt_tokens=352,
|
||||
with_max_tokens_per_query_prompt_tokens=223,
|
||||
with_max_tokens_per_doc_prompt_tokens=221,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=92,
|
||||
),
|
||||
# 3. bi-encoder
|
||||
TestConfig(
|
||||
model="intfloat/multilingual-e5-small",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
# This model has produced both prompt-token totals in CI/local cache;
|
||||
# keep truncation checks exact while tolerating the boundary delta.
|
||||
without_truncated_prompt_tokens=(284, 285),
|
||||
with_max_tokens_per_query_prompt_tokens=(154, 155),
|
||||
with_max_tokens_per_doc_prompt_tokens=154,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=24,
|
||||
),
|
||||
# 4. late-interaction
|
||||
TestConfig(
|
||||
model="answerdotai/answerai-colbert-small-v1",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
without_truncated_prompt_tokens=284,
|
||||
with_max_tokens_per_query_prompt_tokens=154,
|
||||
with_max_tokens_per_doc_prompt_tokens=154,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=24,
|
||||
),
|
||||
# 5. jinaai/jina-reranker-v3
|
||||
TestConfig(
|
||||
model="jinaai/jina-reranker-v3",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
without_truncated_prompt_tokens=567,
|
||||
with_max_tokens_per_query_prompt_tokens=308,
|
||||
with_max_tokens_per_doc_prompt_tokens=436,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=177,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def assert_prompt_tokens(actual: int, expected: ExpectedPromptTokens) -> None:
|
||||
if isinstance(expected, int):
|
||||
assert actual == expected
|
||||
else:
|
||||
assert actual in expected
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=RERANK_CONFIGS, ids=lambda c: c.model)
|
||||
def server(request):
|
||||
config: TestConfig = request.param
|
||||
env_dict = {"VLLM_LOGGING_LEVEL": "WARNING"}
|
||||
with RemoteOpenAIServer(
|
||||
config.model, config.args, env_dict=env_dict
|
||||
) as remote_server:
|
||||
yield config, remote_server
|
||||
|
||||
|
||||
def test_without_truncated(server):
|
||||
"""Test that max_tokens_per_doc truncates documents correctly."""
|
||||
config, remote_server = server
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={"model": config.model, "query": long_query, "documents": [long_doc]},
|
||||
)
|
||||
response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 1
|
||||
assert_prompt_tokens(
|
||||
rerank.usage.prompt_tokens,
|
||||
config.without_truncated_prompt_tokens,
|
||||
)
|
||||
|
||||
|
||||
def test_max_tokens_per_query(server):
|
||||
"""Test that max_tokens_per_doc truncates documents correctly."""
|
||||
config, remote_server = server
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": config.model,
|
||||
"query": long_query,
|
||||
"documents": [long_doc],
|
||||
"max_tokens_per_query": 10,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 1
|
||||
assert_prompt_tokens(
|
||||
rerank.usage.prompt_tokens,
|
||||
config.with_max_tokens_per_query_prompt_tokens,
|
||||
)
|
||||
|
||||
|
||||
def test_max_tokens_per_doc(server):
|
||||
"""Test that max_tokens_per_doc truncates documents correctly."""
|
||||
config, remote_server = server
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": config.model,
|
||||
"query": long_query,
|
||||
"documents": [long_doc],
|
||||
"max_tokens_per_doc": 10,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 1
|
||||
assert_prompt_tokens(
|
||||
rerank.usage.prompt_tokens,
|
||||
config.with_max_tokens_per_doc_prompt_tokens,
|
||||
)
|
||||
|
||||
|
||||
def test_max_tokens_per_query_and_doc(server):
|
||||
"""Test that max_tokens_per_doc truncates documents correctly."""
|
||||
config, remote_server = server
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": config.model,
|
||||
"query": long_query,
|
||||
"documents": [long_doc],
|
||||
"max_tokens_per_query": 10,
|
||||
"max_tokens_per_doc": 10,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 1
|
||||
assert_prompt_tokens(
|
||||
rerank.usage.prompt_tokens,
|
||||
config.with_max_tokens_per_query_and_doc_prompt_tokens,
|
||||
)
|
||||
@@ -1,104 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.config.pooler import PoolerConfig
|
||||
|
||||
|
||||
def test_idefics_multimodal(
|
||||
vllm_runner,
|
||||
) -> None:
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
with vllm_runner(
|
||||
model_name="HuggingFaceM4/Idefics3-8B-Llama3",
|
||||
runner="pooling",
|
||||
convert="classify",
|
||||
load_format="dummy",
|
||||
max_model_len=512,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=1,
|
||||
disable_log_stats=True,
|
||||
dtype="bfloat16",
|
||||
) as vllm_model:
|
||||
llm = vllm_model.get_llm()
|
||||
outputs = llm.classify(prompts)
|
||||
for output in outputs:
|
||||
assert len(output.outputs.probs) == 2
|
||||
|
||||
|
||||
def update_config(config):
|
||||
text_config = config.get_text_config()
|
||||
text_config.update(
|
||||
{
|
||||
"architectures": ["Gemma3ForSequenceClassification"],
|
||||
"classifier_from_token": ["A", "B", "C", "D", "E"],
|
||||
"method": "no_post_processing",
|
||||
"id2label": {
|
||||
"A": "Chair",
|
||||
"B": "Couch",
|
||||
"C": "Table",
|
||||
"D": "Bed",
|
||||
"E": "Cupboard",
|
||||
},
|
||||
}
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def test_gemma_multimodal(
|
||||
vllm_runner,
|
||||
) -> None:
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": """
|
||||
You are a helpful assistant. You will be given a product description
|
||||
which may also include an image. Classify the following product into
|
||||
one of the categories:
|
||||
|
||||
A = chair
|
||||
B = couch
|
||||
C = table
|
||||
D = bed
|
||||
E = cupboard
|
||||
|
||||
You'll answer with exactly one letter (A, B, C, D, or E).""",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/red_chair.jpg"
|
||||
},
|
||||
},
|
||||
{"type": "text", "text": "A fine 19th century piece of furniture."},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
with vllm_runner(
|
||||
model_name="google/gemma-3-4b-it",
|
||||
runner="pooling",
|
||||
convert="classify",
|
||||
load_format="auto",
|
||||
hf_overrides=update_config,
|
||||
pooler_config=PoolerConfig(seq_pooling_type="LAST"),
|
||||
max_model_len=512,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=1,
|
||||
disable_log_stats=True,
|
||||
dtype="bfloat16",
|
||||
) as vllm_model:
|
||||
llm = vllm_model.get_llm()
|
||||
prompts = llm._preprocess_chat([messages])
|
||||
|
||||
result = llm.classify(prompts)
|
||||
assert result[0].outputs.probs[0] > 0.95
|
||||
assert all(c < 0.05 for c in result[0].outputs.probs[1:])
|
||||
@@ -1,44 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
from tests.models.utils import check_embeddings_close
|
||||
from vllm.config import PoolerConfig
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["BAAI/bge-m3"],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@torch.inference_mode
|
||||
def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype: str):
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
pooler_config=PoolerConfig(task="token_embed"),
|
||||
max_model_len=None,
|
||||
dtype=dtype,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_embed(example_prompts)
|
||||
|
||||
with hf_runner(model, auto_cls=AutoModel, dtype=dtype) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
embedding = output.last_hidden_state[0].float()
|
||||
hf_outputs.append(embedding.cpu())
|
||||
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_output,
|
||||
embeddings_1_lst=vllm_output,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
@@ -1,34 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForSequenceClassification
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["Rami/multi-label-class-classification-on-github-issues"],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_classify_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.classify(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.classify(example_prompts)
|
||||
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = torch.tensor(hf_output)
|
||||
vllm_output = torch.tensor(vllm_output)
|
||||
|
||||
assert torch.allclose(
|
||||
hf_output, vllm_output, 1e-3 if dtype == "float" else 1e-2
|
||||
)
|
||||
@@ -1,129 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: SIM117
|
||||
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
|
||||
from ...utils import EmbedModelInfo
|
||||
|
||||
MODELS = [
|
||||
EmbedModelInfo(
|
||||
"nomic-ai/nomic-embed-text-v1",
|
||||
# Fixme:
|
||||
# Update nomic-embed code to support the latest
|
||||
# HF version and remove revision set.
|
||||
revision="720244025c1a7e15661a174c63cce63c8218e52b",
|
||||
),
|
||||
# EmbedModelInfo("nomic-ai/nomic-embed-text-v1.5"),
|
||||
# EmbedModelInfo("nomic-ai/CodeRankEmbed"),
|
||||
EmbedModelInfo("nomic-ai/nomic-embed-text-v2-moe"),
|
||||
# EmbedModelInfo("Snowflake/snowflake-arctic-embed-m-long"),
|
||||
]
|
||||
|
||||
rope_theta = 1000
|
||||
factor = 4.0
|
||||
original_max_position_embeddings = 2048
|
||||
max_model_len = int(original_max_position_embeddings * factor)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def predownload_models():
|
||||
"""Warm the HF cache so engine startup never hits transient hub errors."""
|
||||
from vllm.transformers_utils.repo_utils import hf_api, with_retry
|
||||
|
||||
for model_info in MODELS:
|
||||
with_retry(
|
||||
partial(
|
||||
hf_api().snapshot_download,
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
),
|
||||
f"Error downloading {model_info.name}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_default(model_info, vllm_runner):
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=None,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
if model_info.name == "nomic-ai/nomic-embed-text-v2-moe":
|
||||
# For nomic-embed-text-v2-moe the length is set to 512
|
||||
# by sentence_bert_config.json.
|
||||
assert model_config.max_model_len == 512
|
||||
if model_info.name == "nomic-ai/nomic-embed-text-v1":
|
||||
assert model_config.max_model_len == 8192
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_set_max_model_len_legal(model_info, vllm_runner):
|
||||
# set max_model_len <= 512
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=256,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
assert model_config.max_model_len == 256
|
||||
|
||||
# For nomic-embed-text-v2-moe the length is set to 512
|
||||
# by sentence_bert_config.json.
|
||||
if model_info.name == "nomic-ai/nomic-embed-text-v2-moe":
|
||||
with pytest.raises(ValueError):
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=1024,
|
||||
):
|
||||
pass
|
||||
return
|
||||
|
||||
# set 512 < max_model_len <= 2048
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=1024,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
assert model_config.max_model_len == 1024
|
||||
|
||||
# set max_model_len > 2048
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=4096,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
assert model_config.max_model_len == 4096
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_use_rope_scaling_legal(model_info, vllm_runner):
|
||||
hf_overrides = {
|
||||
"rope_parameters": {
|
||||
"rope_theta": rope_theta,
|
||||
"rope_type": "yarn",
|
||||
"factor": factor,
|
||||
"original_max_position_embeddings": original_max_position_embeddings,
|
||||
},
|
||||
"max_model_len": max_model_len,
|
||||
}
|
||||
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=None,
|
||||
hf_overrides=hf_overrides,
|
||||
):
|
||||
pass
|
||||
@@ -1,167 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.models.utils import softmax
|
||||
from vllm.config import PoolerConfig
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["jason9693/Qwen2.5-1.5B-apeach", "papluca/xlm-roberta-base-language-detection"],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_classify_models_using_activation(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=False),
|
||||
) as vllm_model:
|
||||
wo_activation_out = vllm_model.classify(example_prompts)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=True),
|
||||
) as vllm_model:
|
||||
w_activation_out = vllm_model.classify(example_prompts)
|
||||
|
||||
for wo_activation, w_activation in zip(wo_activation_out, w_activation_out):
|
||||
wo_activation = torch.tensor(wo_activation)
|
||||
w_activation = torch.tensor(w_activation)
|
||||
|
||||
assert not torch.allclose(wo_activation, w_activation, atol=1e-2), (
|
||||
"pooler_config is not working"
|
||||
)
|
||||
assert torch.allclose(
|
||||
softmax(wo_activation), w_activation, 1e-3 if dtype == "float" else 1e-2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"intfloat/multilingual-e5-small",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_embed_models_using_normalize(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=False),
|
||||
) as vllm_model:
|
||||
wo_normalize = torch.tensor(vllm_model.embed(example_prompts))
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=True),
|
||||
) as vllm_model:
|
||||
w_normalize = torch.tensor(vllm_model.embed(example_prompts))
|
||||
|
||||
assert not torch.allclose(wo_normalize, w_normalize, atol=1e-2), (
|
||||
"pooler_config normalize is not working"
|
||||
)
|
||||
assert torch.allclose(
|
||||
F.normalize(wo_normalize, p=2, dim=-1), w_normalize, atol=1e-2
|
||||
), "w_normal should be close to normal(wo_normal)."
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"internlm/internlm2-1_8b-reward",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_reward_models_using_activation(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=1024,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=False),
|
||||
) as vllm_model:
|
||||
wo_activation = vllm_model.token_classify(example_prompts)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=1024,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=True),
|
||||
) as vllm_model:
|
||||
w_activation = vllm_model.token_classify(example_prompts)
|
||||
|
||||
for wo, w in zip(wo_activation, w_activation):
|
||||
wo = torch.tensor(wo)
|
||||
w = torch.tensor(w)
|
||||
|
||||
assert not torch.allclose(wo, w, atol=1e-2), (
|
||||
"pooler_config activation is not working"
|
||||
)
|
||||
assert torch.allclose(softmax(wo), w, atol=1e-2), (
|
||||
"w_activation should be close to activation(wo_activation)."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"intfloat/multilingual-e5-small",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_multi_vector_retrieval_models_using_normalize(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=False, task="token_embed"),
|
||||
) as vllm_model:
|
||||
wo_normalize = vllm_model.token_embed(example_prompts)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=True, task="token_embed"),
|
||||
) as vllm_model:
|
||||
w_normalize = vllm_model.token_embed(example_prompts)
|
||||
|
||||
for wo, w in zip(wo_normalize, w_normalize):
|
||||
assert not torch.allclose(wo, w, atol=1e-2), (
|
||||
"pooler_config normalize is not working"
|
||||
)
|
||||
assert torch.allclose(F.normalize(wo, p=2, dim=-1), w, atol=1e-2), (
|
||||
"w_normal should be close to normal(wo_normal)."
|
||||
)
|
||||
@@ -1,179 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from transformers import AutoModel
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_constants import MiB_bytes
|
||||
|
||||
from ....conftest import HfRunner
|
||||
from ....utils import VLLM_PATH
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from _typeshed import StrPath
|
||||
|
||||
|
||||
FIXTURES_PATH = VLLM_PATH / "tests/models/fixtures"
|
||||
assert FIXTURES_PATH.exists()
|
||||
FIXTURE_REWARD_RESULT = {
|
||||
"Qwen/Qwen2.5-Math-PRM-7B": FIXTURES_PATH / "qwen2_5_math_prm_reward_step.json",
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def math_step_prompts():
|
||||
# ruff: noqa: E501
|
||||
data = {
|
||||
"system": "Please reason step by step, and put your final answer within \\boxed{}. ",
|
||||
"query": "Sue lives in a fun neighborhood. One weekend, the neighbors decided to play a prank on Sue. On Friday morning, the neighbors placed 18 pink plastic flamingos out on Sue's front yard. On Saturday morning, the neighbors took back one third of the flamingos, painted them white, and put these newly painted white flamingos back out on Sue's front yard. Then, on Sunday morning, they added another 18 pink plastic flamingos to the collection. At noon on Sunday, how many more pink plastic flamingos were out than white plastic flamingos?",
|
||||
"response": [
|
||||
"To find out how many more pink plastic flamingos were out than white plastic flamingos at noon on Sunday, we can break down the problem into steps. First, on Friday, the neighbors start with 18 pink plastic flamingos.",
|
||||
"On Saturday, they take back one third of the flamingos. Since there were 18 flamingos, (1/3 \\times 18 = 6) flamingos are taken back. So, they have (18 - 6 = 12) flamingos left in their possession. Then, they paint these 6 flamingos white and put them back out on Sue's front yard. Now, Sue has the original 12 pink flamingos plus the 6 new white ones. Thus, by the end of Saturday, Sue has (12 + 6 = 18) pink flamingos and 6 white flamingos.",
|
||||
"On Sunday, the neighbors add another 18 pink plastic flamingos to Sue's front yard. By the end of Sunday morning, Sue has (18 + 18 = 36) pink flamingos and still 6 white flamingos.",
|
||||
"To find the difference, subtract the number of white flamingos from the number of pink flamingos: (36 - 6 = 30). Therefore, at noon on Sunday, there were 30 more pink plastic flamingos out than white plastic flamingos. The answer is (\\boxed{30}).",
|
||||
],
|
||||
}
|
||||
answer = "<extra_0>".join(data["response"]) + "<extra_0>"
|
||||
prompt = f"<im_start>system\n{data['system']}<im_end>\n<im_start>user\n{data['query']}<im_end>\n<im_start>assistant\n{answer}<im_end><|endoftext|>"
|
||||
return [prompt]
|
||||
|
||||
|
||||
def step_reward_patch_hf_model(hf_model: HfRunner):
|
||||
# Patch the hf_runner to use the step reward function
|
||||
def make_step_rewards(
|
||||
logits: torch.Tensor, token_masks: torch.Tensor
|
||||
) -> list[list[float]]:
|
||||
probabilities = F.softmax(logits, dim=-1)
|
||||
probabilities = probabilities * token_masks.unsqueeze(-1)
|
||||
|
||||
all_scores_res: list[list[float]] = []
|
||||
for i in range(probabilities.size(0)):
|
||||
sample = probabilities[i] # seq_len, num_labels
|
||||
positive_probs = sample[sample != 0].view(-1, 2)
|
||||
non_zero_elements_list = positive_probs.cpu().tolist()
|
||||
all_scores_res.append(non_zero_elements_list)
|
||||
return all_scores_res
|
||||
|
||||
def reward(prompts: list[str]) -> list[list[float]]:
|
||||
input_ids = hf_model.tokenizer(prompts, return_tensors="pt").input_ids
|
||||
input_ids = hf_model.wrap_device(input_ids)
|
||||
outputs = hf_model.model(input_ids=input_ids)
|
||||
|
||||
step_sep_id = hf_model.tokenizer.encode("<extra_0>")[0]
|
||||
token_masks = input_ids == step_sep_id
|
||||
return make_step_rewards(outputs[0], token_masks)
|
||||
|
||||
hf_model.reward = reward # type: ignore[attr-defined]
|
||||
|
||||
return hf_model
|
||||
|
||||
|
||||
def dump_reward_outputs(outputs: list[list[float]], filename: "StrPath"):
|
||||
with open(filename, "w", encoding="utf-8") as f:
|
||||
json.dump(outputs, f)
|
||||
|
||||
|
||||
def load_reward_outputs(filename: "StrPath") -> list[list[float]]:
|
||||
with open(filename, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
pytest.param(
|
||||
"Qwen/Qwen2.5-Math-PRM-7B",
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_prm_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
math_step_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
if current_platform.is_cpu():
|
||||
pytest.skip("CPU only supports V1")
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=1024,
|
||||
dtype=dtype,
|
||||
kv_cache_memory_bytes=64 * MiB_bytes,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.token_classify(math_step_prompts)
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
|
||||
hf_model = step_reward_patch_hf_model(hf_model)
|
||||
hf_outputs = hf_model.reward(math_step_prompts)
|
||||
|
||||
dump_reward_outputs(
|
||||
hf_outputs,
|
||||
FIXTURE_REWARD_RESULT[model],
|
||||
)
|
||||
|
||||
# check logits difference
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = torch.tensor(hf_output).float()
|
||||
vllm_output = torch.tensor(vllm_output).float()
|
||||
|
||||
assert torch.allclose(hf_output, vllm_output, 1.5e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
pytest.param(
|
||||
"Qwen/Qwen2.5-Math-PRM-7B",
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_prm_models_with_golden_outputs(
|
||||
vllm_runner,
|
||||
math_step_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
if not FIXTURE_REWARD_RESULT.get(model):
|
||||
pytest.skip(f"No available golden outputs for {model}.")
|
||||
|
||||
use_v2_model_runner = not current_platform.is_cpu()
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2_model_runner else "0")
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=1024,
|
||||
dtype=dtype,
|
||||
kv_cache_memory_bytes=64 * MiB_bytes,
|
||||
) as vllm_model:
|
||||
assert (
|
||||
vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
is use_v2_model_runner
|
||||
)
|
||||
vllm_outputs = vllm_model.token_classify(math_step_prompts)
|
||||
|
||||
golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
|
||||
|
||||
# check logits difference
|
||||
for golden_output, vllm_output in zip(golden_outputs, vllm_outputs):
|
||||
golden_output = torch.tensor(golden_output).float()
|
||||
vllm_output = torch.tensor(vllm_output).float()
|
||||
|
||||
assert torch.allclose(golden_output, vllm_output, 1.5e-2)
|
||||
@@ -1,281 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.models.bert import (
|
||||
BertMLMHead,
|
||||
SPLADESparsePooler,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.v1.pool.late_interaction_runner import LateInteractionRunner
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Functional test: SPLADE formula correctness (no HF download needed)
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("B,T,H,V", [(2, 3, 5, 7)])
|
||||
@torch.inference_mode
|
||||
def test_splade_pooler_matches_reference_formula(B, T, H, V):
|
||||
"""Ensure SPLADESparsePooler forward() matches the mathematical formula:
|
||||
log1p(relu(logits)) -> max over sequence length (after masking)."""
|
||||
torch.manual_seed(0)
|
||||
|
||||
# Prepare [B] sequences of shape [T, H]
|
||||
hs_list = [torch.randn(T, H) for _ in range(B)]
|
||||
hs_tenser = torch.cat(hs_list)
|
||||
|
||||
# Simulate PoolingMetadata (only required fields)
|
||||
prompt_lens = [T, T - 1]
|
||||
prompt_lens_tenser = torch.tensor(prompt_lens, dtype=torch.int32)
|
||||
token_ids = torch.tensor(
|
||||
[
|
||||
[101, 5, 102], # Batch 0: [CLS], token, [SEP]
|
||||
[101, 6, 6], # Batch 1: [CLS], token, token (last token ignored)
|
||||
],
|
||||
dtype=torch.long,
|
||||
)
|
||||
meta = PoolingMetadata(
|
||||
prompt_lens=prompt_lens_tenser,
|
||||
prompt_token_ids=token_ids,
|
||||
prompt_token_ids_cpu=token_ids,
|
||||
pooling_params=[PoolingParams(task="embed")] * B,
|
||||
pooling_states=[PoolingStates() for _ in range(B)],
|
||||
)
|
||||
|
||||
# MLM head (prefer BertMLMHead, fallback to Linear if unavailable)
|
||||
try:
|
||||
mlm_head = BertMLMHead(hidden_size=H, vocab_size=V, layer_norm_eps=1e-12)
|
||||
except Exception:
|
||||
mlm_head = nn.Linear(H, V, bias=True)
|
||||
|
||||
# Forward pass through SPLADE pooler
|
||||
pooler = SPLADESparsePooler(mlm_head=mlm_head, pooling="max", remove_cls_sep=True)
|
||||
pooled = pooler(hidden_states=hs_tenser, pooling_metadata=meta) # list of [V]
|
||||
|
||||
# Basic output checks
|
||||
assert isinstance(pooled, torch.Tensor) and len(pooled) == B
|
||||
for vec in pooled:
|
||||
assert vec.shape == (V,)
|
||||
assert torch.isfinite(vec).all()
|
||||
assert (vec >= 0).all(), "SPLADE outputs must be non-negative."
|
||||
|
||||
# Reference implementation for comparison
|
||||
def ref_one(hs: torch.Tensor, L: int, tid_row: torch.Tensor) -> torch.Tensor:
|
||||
keep = torch.ones(L, dtype=torch.bool)
|
||||
if L > 0 and tid_row[0].item() == 101: # remove CLS
|
||||
keep[0] = False
|
||||
if L > 0 and tid_row[L - 1].item() == 102: # remove SEP
|
||||
keep[L - 1] = False
|
||||
|
||||
valid = hs[:L][keep[:L]]
|
||||
if valid.numel() == 0:
|
||||
return torch.zeros(V, dtype=torch.float32)
|
||||
|
||||
logits = mlm_head(valid) # [L', V]
|
||||
scores = torch.log1p(torch.relu(logits)) # [L', V]
|
||||
return scores.max(dim=0).values.to(torch.float32)
|
||||
|
||||
torch.testing.assert_close(
|
||||
pooled[0],
|
||||
ref_one(hs_list[0], prompt_lens[0], token_ids[0]),
|
||||
rtol=1e-4,
|
||||
atol=1e-4,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
pooled[1],
|
||||
ref_one(hs_list[1], prompt_lens[1], token_ids[1]),
|
||||
rtol=1e-4,
|
||||
atol=1e-4,
|
||||
)
|
||||
|
||||
|
||||
def test_pooling_runner_gathers_required_token_ids() -> None:
|
||||
runner = PoolingRunner.__new__(PoolingRunner)
|
||||
pooling_params = PoolingParams(task="embed", requires_token_ids=True)
|
||||
runner.pooling_params = {1: pooling_params, 3: pooling_params}
|
||||
runner.pooling_states = {1: PoolingStates(), 3: PoolingStates()}
|
||||
runner.prompt_token_ids = {
|
||||
1: torch.tensor([101, 102]),
|
||||
3: torch.tensor([101, 11, 102]),
|
||||
}
|
||||
|
||||
input_batch = MagicMock(spec=InputBatch)
|
||||
input_batch.idx_mapping_np = np.array([3, 1], dtype=np.int32)
|
||||
input_batch.num_reqs = 2
|
||||
req_states = MagicMock(spec=RequestState)
|
||||
req_states.prompt_len = MagicMock(np=np.array([0, 2, 0, 3], dtype=np.int32))
|
||||
metadata = runner._get_pooling_metadata(
|
||||
input_batch, req_states, torch.device("cpu")
|
||||
)
|
||||
|
||||
expected = torch.tensor([[101, 11, 102], [101, 102, 0]])
|
||||
assert metadata.prompt_token_ids_cpu is not None
|
||||
assert metadata.prompt_token_ids is None
|
||||
assert metadata.prompt_token_ids_cpu.is_pinned() is False
|
||||
torch.testing.assert_close(
|
||||
metadata.prompt_lens, torch.tensor([3, 2], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(metadata.prompt_token_ids_cpu, expected)
|
||||
|
||||
|
||||
def test_pooling_runner_stores_only_required_token_ids() -> None:
|
||||
runner = PoolingRunner.__new__(PoolingRunner)
|
||||
runner.model = MagicMock()
|
||||
runner.supported_tasks = frozenset({"embed"})
|
||||
runner.pooling_params = {}
|
||||
runner.pooling_states = {}
|
||||
runner.prompt_token_ids = {}
|
||||
runner.late_interaction_runner = MagicMock()
|
||||
|
||||
runner.add_request("req-1", 1, PoolingParams(task="embed"), [101, 102])
|
||||
runner.add_request(
|
||||
"req-2",
|
||||
2,
|
||||
PoolingParams(task="embed", requires_token_ids=True),
|
||||
[101, 11, 102],
|
||||
)
|
||||
|
||||
assert 1 not in runner.prompt_token_ids
|
||||
torch.testing.assert_close(runner.prompt_token_ids[2], torch.tensor([101, 11, 102]))
|
||||
|
||||
|
||||
def test_pooling_runner_releases_aborted_late_interaction_doc() -> None:
|
||||
runner = PoolingRunner.__new__(PoolingRunner)
|
||||
runner.late_interaction_runner = LateInteractionRunner()
|
||||
|
||||
query_key = "query-abort"
|
||||
late_interaction_runner = runner.late_interaction_runner
|
||||
late_interaction_runner._query_cache[query_key] = torch.ones(2, 4)
|
||||
late_interaction_runner._query_uses[query_key] = 1
|
||||
late_interaction_runner._doc_query_keys["doc-req"] = query_key
|
||||
|
||||
runner.on_requests_finished({"doc-req"})
|
||||
|
||||
assert not late_interaction_runner._query_cache
|
||||
assert not late_interaction_runner._query_uses
|
||||
assert not late_interaction_runner._doc_query_keys
|
||||
|
||||
|
||||
def test_encoder_cache_reset_clears_late_interaction_state() -> None:
|
||||
# Cached query embeddings are only invalidated by weight reloads, which
|
||||
# reach the runner via reset_encoder_cache. Resetting the multi-modal
|
||||
# cache is unrelated and must leave them intact.
|
||||
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.encoder_cache = MagicMock()
|
||||
runner.pooling_runner = MagicMock()
|
||||
|
||||
runner.reset_mm_cache()
|
||||
|
||||
runner.encoder_cache.reset_mm_cache.assert_called_once_with()
|
||||
runner.pooling_runner.clear.assert_not_called()
|
||||
|
||||
runner.reset_encoder_cache()
|
||||
|
||||
runner.encoder_cache.reset_encoder_cache.assert_called_once_with()
|
||||
runner.pooling_runner.clear.assert_called_once_with()
|
||||
|
||||
|
||||
def test_pooling_runner_rejects_unsupported_selected_task() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "plugin"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "encoder_only"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "plugin"
|
||||
|
||||
with (
|
||||
patch.object(PoolingRunner, "get_supported_tasks", return_value=["embed"]),
|
||||
pytest.raises(ValueError, match="selects 'plugin'"),
|
||||
):
|
||||
PoolingRunner(model, vllm_config)
|
||||
|
||||
|
||||
def test_pooling_runner_supports_encoder_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "encoder_only"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "token_classify"
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"token_classify"}
|
||||
|
||||
|
||||
def test_pooling_runner_supports_encoder_token_embedding() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"token_embed"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "encoder_only"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "token_embed"
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"token_embed"}
|
||||
|
||||
|
||||
def test_pooling_runner_supports_decoder_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "decoder"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "token_classify"
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"token_classify"}
|
||||
|
||||
|
||||
def test_pooling_runner_keeps_decoder_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "decoder"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "embed"
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"embed", "token_classify"}
|
||||
|
||||
|
||||
def test_pooling_runner_keeps_decoder_token_embedding() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "token_embed"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "decoder"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "embed"
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"embed", "token_embed"}
|
||||
|
||||
|
||||
def test_pooling_runner_supports_embed_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "embed&token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
|
||||
vllm_config.model_config.attn_type = "encoder_only"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "embed&token_classify"
|
||||
assert PoolingRunner(model, vllm_config).supported_tasks == {
|
||||
"embed",
|
||||
"embed&token_classify",
|
||||
}
|
||||
@@ -1,280 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForMaskedLM, AutoModelForTokenClassification
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from tests.models.utils import softmax
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def seed_everything():
|
||||
"""Seed all random number generators for reproducibility."""
|
||||
seed = 0
|
||||
set_random_seed(seed)
|
||||
torch.backends.cudnn.deterministic = True
|
||||
torch.backends.cudnn.benchmark = False
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"boltuix/NeuroBERT-NER",
|
||||
],
|
||||
)
|
||||
# The float32 is required for this tiny model to pass the test.
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@pytest.mark.core_model
|
||||
@torch.inference_mode
|
||||
def test_bert_model_runner_v2(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
prompt_batches = [[example_prompts[0]], example_prompts]
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_output_batches = [
|
||||
vllm_model.token_classify(prompts) for prompts in prompt_batches
|
||||
]
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=AutoModelForTokenClassification,
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
hf_output_batches = [[hf_outputs[0]], hf_outputs]
|
||||
|
||||
# check logits difference
|
||||
for hf_outputs, vllm_outputs in zip(hf_output_batches, vllm_output_batches):
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["disham993/electrical-ner-ModernBERT-base"])
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
@torch.inference_mode
|
||||
def test_modernbert_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
# NOTE: https://github.com/vllm-project/vllm/pull/32403
|
||||
# `disham993/electrical-ner-ModernBERT-base` is a randomly initialized
|
||||
# model, which can cause numerical precision variance and edge cases.
|
||||
# We use @flaky(reruns=3) to mitigate intermittent failures.
|
||||
print(
|
||||
f"\n[NOTE] Testing {model} (randomly initialized weights) - "
|
||||
"flaky tolerance enabled due to numerical precision variance."
|
||||
)
|
||||
|
||||
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_classify(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=AutoModelForTokenClassification,
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
|
||||
# check logits difference
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=6e-2, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"])
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
@torch.inference_mode
|
||||
def test_xlm_roberta_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_classify(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=AutoModelForTokenClassification,
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
|
||||
# check logits difference
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
|
||||
|
||||
|
||||
PRIVACY_FILTER_PROMPTS = [
|
||||
"My name is Harry Potter.",
|
||||
"Email me at harry.potter@hogwarts.edu.",
|
||||
"Call me on +44 20 7946 0958 tomorrow.",
|
||||
"My account number is 12345678 and the API key is sk-live-abc123def456.",
|
||||
"I live at 4 Privet Drive, Little Whinging.",
|
||||
"Visit https://example.com/profile/harry for more info.",
|
||||
"We met on 12 January 2024.",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="Workspace growth allocation issue on ROCm. See https://github.com/vllm-project/vllm/issues/48510",
|
||||
)
|
||||
@pytest.mark.parametrize("model", ["openai/privacy-filter"])
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@torch.inference_mode
|
||||
def test_openai_privacy_filter(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_classify(PRIVACY_FILTER_PROMPTS)
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=AutoModelForTokenClassification,
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in PRIVACY_FILTER_PROMPTS:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=0.1, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["bd2lcco/Qwen3-0.6B-finetuned"])
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
@torch.inference_mode
|
||||
def test_auto_conversion(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_classify(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model, dtype=dtype, auto_cls=AutoModelForTokenClassification
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
|
||||
# check logits difference
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
assert torch.allclose(hf_output, vllm_output, atol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
# Original Google checkpoint: legacy `gamma`/`beta` LayerNorm names, an
|
||||
# NSP head (`cls.seq_relationship.*`) and a decoder tied to the input
|
||||
# embeddings (no explicit decoder weight in the checkpoint).
|
||||
"google-bert/bert-base-uncased",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@torch.inference_mode
|
||||
def test_bert_for_masked_lm(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
# BertForMaskedLM exposes its MLM head as a token-level pooling task; the
|
||||
# head applies softmax over the vocabulary, so each output row is a
|
||||
# distribution (matching HF's softmax(logits) below).
|
||||
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_classify(example_prompts)
|
||||
|
||||
# Run hf_runner reference with "highest" fp32 precision to match
|
||||
# default behvior of vLLM. This is needed on ROCm since the
|
||||
# pooling tests set matmul precision to "high" in conftest.py
|
||||
prev_matmul_precision = torch.get_float32_matmul_precision()
|
||||
torch.set_float32_matmul_precision("highest")
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=AutoModelForMaskedLM,
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
torch.set_float32_matmul_precision(prev_matmul_precision)
|
||||
|
||||
# Compare the per-token vocabulary distributions position by position.
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
assert hf_output.shape == vllm_output.shape
|
||||
assert torch.equal(hf_output.argmax(dim=-1), vllm_output.argmax(dim=-1))
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
|
||||
@@ -1,70 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
MODEL_NAME = "sentence-transformers/all-MiniLM-L12-v2"
|
||||
max_model_len = 128
|
||||
|
||||
input_str = """Immerse yourself in the enchanting chronicle of calculus, a
|
||||
mathematical domain that has radically transformed our comprehension of
|
||||
change and motion. Despite its roots in ancient civilizations, the
|
||||
formal birth of calculus predominantly occurred in the 17th century,
|
||||
primarily under the influential guidance of Sir Isaac Newton and Gottfried
|
||||
Wilhelm Leibniz. The earliest traces of calculus concepts are found in
|
||||
ancient Greek mathematics,most notably in the works of Eudoxus and
|
||||
Archimedes, around 300 BCE. They utilized the 'method of exhaustion'—a
|
||||
technique for computing areas and volumes through the use of finite sums.
|
||||
This methodology laid crucial foundational work for integral calculus.
|
||||
In the 17th century, both Newton and Leibniz independently pioneered
|
||||
calculus, each contributing unique perspectives that would shape this new
|
||||
field."""
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def vllm_model(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME, runner="pooling", max_model_len=max_model_len
|
||||
) as model:
|
||||
yield model
|
||||
|
||||
|
||||
def test_smaller_truncation_size(
|
||||
vllm_model,
|
||||
):
|
||||
truncate_prompt_tokens = 10
|
||||
|
||||
vllm_output = vllm_model.llm.embed(
|
||||
input_str,
|
||||
tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
|
||||
)
|
||||
|
||||
prompt_tokens = vllm_output[0].prompt_token_ids
|
||||
|
||||
assert len(prompt_tokens) == truncate_prompt_tokens
|
||||
|
||||
|
||||
def test_max_truncation_size(vllm_model):
|
||||
truncate_prompt_tokens = -1
|
||||
|
||||
vllm_output = vllm_model.llm.embed(
|
||||
input_str,
|
||||
tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
|
||||
)
|
||||
|
||||
prompt_tokens = vllm_output[0].prompt_token_ids
|
||||
|
||||
# Pooling requests emit an embedding rather than generated tokens, so -1
|
||||
# truncates to the full context length.
|
||||
assert len(prompt_tokens) == max_model_len
|
||||
|
||||
|
||||
def test_bigger_truncation_size(vllm_model):
|
||||
truncate_prompt_tokens = max_model_len + 1
|
||||
|
||||
with pytest.raises(VLLMValidationError):
|
||||
vllm_model.llm.embed(
|
||||
input_str,
|
||||
tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
|
||||
)
|
||||
Reference in New Issue
Block a user