[Frontend] support matryoshka representation / support embedding API dimensions (#16331)
This commit is contained in:
+8
-8
@@ -960,19 +960,19 @@ class VllmRunner:
|
||||
req_outputs = self.model.classify(prompts)
|
||||
return [req_output.outputs.probs for req_output in req_outputs]
|
||||
|
||||
def encode(
|
||||
self,
|
||||
prompts: list[str],
|
||||
images: Optional[PromptImageInput] = None,
|
||||
videos: Optional[PromptVideoInput] = None,
|
||||
audios: Optional[PromptAudioInput] = None,
|
||||
) -> list[list[float]]:
|
||||
def encode(self,
|
||||
prompts: list[str],
|
||||
images: Optional[PromptImageInput] = None,
|
||||
videos: Optional[PromptVideoInput] = None,
|
||||
audios: Optional[PromptAudioInput] = None,
|
||||
*args,
|
||||
**kwargs) -> list[list[float]]:
|
||||
inputs = self.get_inputs(prompts,
|
||||
images=images,
|
||||
videos=videos,
|
||||
audios=audios)
|
||||
|
||||
req_outputs = self.model.embed(inputs)
|
||||
req_outputs = self.model.embed(inputs, *args, **kwargs)
|
||||
return [req_output.outputs.embedding for req_output in req_outputs]
|
||||
|
||||
def score(
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""
|
||||
Run `pytest tests/entrypoints/openai/test_embedding_dimensions.py`.
|
||||
"""
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.protocol import EmbeddingResponse
|
||||
|
||||
from ...utils import RemoteOpenAIServer
|
||||
|
||||
|
||||
class ModelInfo(NamedTuple):
|
||||
name: str
|
||||
is_matryoshka: bool
|
||||
|
||||
|
||||
MODELS = [
|
||||
ModelInfo(name="BAAI/bge-m3", is_matryoshka=False),
|
||||
ModelInfo(name="jinaai/jina-embeddings-v3", is_matryoshka=True),
|
||||
]
|
||||
|
||||
input_texts = [
|
||||
"The chef prepared a delicious meal.",
|
||||
] * 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
async def test_validating_dimensions(model: ModelInfo):
|
||||
args = [
|
||||
"--task",
|
||||
"embed",
|
||||
# use half precision for speed and memory savings in CI environment
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--trust_remote_code"
|
||||
]
|
||||
with RemoteOpenAIServer(model.name, args) as remote_server:
|
||||
client = remote_server.get_async_client()
|
||||
|
||||
async def make_request(dimensions):
|
||||
embedding_response = await client.embeddings.create(
|
||||
model=model.name,
|
||||
input=input_texts,
|
||||
dimensions=dimensions,
|
||||
encoding_format="float",
|
||||
)
|
||||
embeddings = EmbeddingResponse.model_validate(
|
||||
embedding_response.model_dump(mode="json"))
|
||||
|
||||
assert embeddings.id is not None
|
||||
assert len(embeddings.data) == 3
|
||||
assert len(embeddings.data[0].embedding) > 0
|
||||
assert embeddings.usage.completion_tokens == 0
|
||||
assert embeddings.usage.prompt_tokens > 0
|
||||
assert embeddings.usage.total_tokens > 0
|
||||
|
||||
if dimensions is not None:
|
||||
assert len(embeddings.data[0].embedding) == dimensions
|
||||
|
||||
if model.is_matryoshka:
|
||||
for dimensions in [None, 16]:
|
||||
await make_request(dimensions)
|
||||
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
for dimensions in [-1]:
|
||||
await make_request(dimensions)
|
||||
|
||||
else:
|
||||
for dimensions in [None]:
|
||||
await make_request(dimensions)
|
||||
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
for dimensions in [-1, 16]:
|
||||
await make_request(dimensions)
|
||||
@@ -8,7 +8,8 @@ import math
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.embedding.utils import check_embeddings_close
|
||||
from tests.models.embedding.utils import check_embeddings_close, matryoshka_fy
|
||||
from vllm import PoolingParams
|
||||
|
||||
SCORING_MODELS = [
|
||||
"jinaai/jina-reranker-v2-base-multilingual", # Roberta
|
||||
@@ -126,3 +127,40 @@ def test_embeddings(
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", EMBEDDING_MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("dimensions", [16, 32])
|
||||
def test_matryoshka(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model,
|
||||
dtype: str,
|
||||
dimensions: int,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
|
||||
example_prompts = EMBEDDING_PROMPTS
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
is_sentence_transformer=True,
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.encode(example_prompts, task="text-matching")
|
||||
hf_outputs = matryoshka_fy(hf_outputs, dimensions)
|
||||
|
||||
with vllm_runner(model, task="embed", dtype=dtype,
|
||||
max_model_len=None) as vllm_model:
|
||||
vllm_outputs = vllm_model.encode(
|
||||
example_prompts,
|
||||
pooling_params=PoolingParams(dimensions=dimensions))
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
@@ -30,3 +30,10 @@ def check_embeddings_close(
|
||||
f"\n{name_1}:\t{embeddings_1[:16]!r}")
|
||||
|
||||
assert sim >= 1 - tol, fail_msg
|
||||
|
||||
|
||||
def matryoshka_fy(tensor, dimensions):
|
||||
tensor = torch.tensor(tensor)
|
||||
tensor = tensor[..., :dimensions]
|
||||
tensor = F.normalize(tensor, p=2, dim=1)
|
||||
return tensor
|
||||
|
||||
Reference in New Issue
Block a user