Signed-off-by: Abhijit <abroy@redhat.com> Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -364,6 +364,7 @@ class HfRunner:
|
||||
model_name: str,
|
||||
dtype: str = "auto",
|
||||
*,
|
||||
revision: str | None = None,
|
||||
model_kwargs: dict[str, Any] | None = None,
|
||||
trust_remote_code: bool = True,
|
||||
is_sentence_transformer: bool = False,
|
||||
@@ -383,6 +384,7 @@ class HfRunner:
|
||||
self._init(
|
||||
model_name=model_name,
|
||||
dtype=dtype,
|
||||
revision=revision,
|
||||
model_kwargs=model_kwargs,
|
||||
trust_remote_code=trust_remote_code,
|
||||
is_sentence_transformer=is_sentence_transformer,
|
||||
@@ -396,6 +398,7 @@ class HfRunner:
|
||||
model_name: str,
|
||||
dtype: str = "auto",
|
||||
*,
|
||||
revision: str | None = None,
|
||||
model_kwargs: dict[str, Any] | None = None,
|
||||
trust_remote_code: bool = True,
|
||||
is_sentence_transformer: bool = False,
|
||||
@@ -437,6 +440,7 @@ class HfRunner:
|
||||
|
||||
self.model = SentenceTransformer(
|
||||
model_name,
|
||||
revision=revision,
|
||||
device=self.device,
|
||||
model_kwargs=model_kwargs,
|
||||
trust_remote_code=trust_remote_code,
|
||||
@@ -447,6 +451,7 @@ class HfRunner:
|
||||
|
||||
self.model = CrossEncoder(
|
||||
model_name,
|
||||
revision=revision,
|
||||
device=self.device,
|
||||
automodel_args=model_kwargs,
|
||||
trust_remote_code=trust_remote_code,
|
||||
@@ -456,6 +461,7 @@ class HfRunner:
|
||||
nn.Module,
|
||||
auto_cls.from_pretrained(
|
||||
model_name,
|
||||
revision=revision,
|
||||
trust_remote_code=trust_remote_code,
|
||||
**model_kwargs,
|
||||
),
|
||||
|
||||
@@ -74,10 +74,25 @@ class MtebEmbedMixin(mteb.EncoderProtocol):
|
||||
return sim
|
||||
|
||||
|
||||
class HfMtebEncoder(MtebEmbedMixin):
|
||||
def __init__(self, model):
|
||||
self.model = model
|
||||
|
||||
def encode(
|
||||
self,
|
||||
inputs: DataLoader[mteb.types.BatchedInput],
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> np.ndarray:
|
||||
sentences = [text for batch in inputs for text in batch["text"]]
|
||||
return self.model.encode(sentences)
|
||||
|
||||
|
||||
class VllmMtebEncoder(MtebEmbedMixin):
|
||||
def __init__(self, vllm_model):
|
||||
def __init__(self, vllm_model, prompt_prefix: str | None = None):
|
||||
self.llm = vllm_model
|
||||
self.rng = np.random.default_rng(seed=42)
|
||||
self.prompt_prefix = prompt_prefix
|
||||
|
||||
def encode(
|
||||
self,
|
||||
@@ -87,7 +102,11 @@ class VllmMtebEncoder(MtebEmbedMixin):
|
||||
) -> np.ndarray:
|
||||
# Hoping to discover potential scheduling
|
||||
# issues by randomizing the order.
|
||||
sentences = [text for batch in inputs for text in batch["text"]]
|
||||
sentences = [
|
||||
self.prompt_prefix + text if self.prompt_prefix else text
|
||||
for batch in inputs
|
||||
for text in batch["text"]
|
||||
]
|
||||
r = self.rng.permutation(len(sentences))
|
||||
sentences = [sentences[i] for i in r]
|
||||
outputs = self.llm.embed(sentences, use_tqdm=False)
|
||||
@@ -143,6 +162,7 @@ def mteb_test_embed_models(
|
||||
vllm_extra_kwargs=None,
|
||||
hf_model_callback=None,
|
||||
atol=MTEB_EMBED_TOL,
|
||||
prompt_prefix: str | None = None,
|
||||
):
|
||||
vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs)
|
||||
|
||||
@@ -182,7 +202,7 @@ def mteb_test_embed_models(
|
||||
)
|
||||
|
||||
vllm_main_score = run_mteb_embed_task(
|
||||
VllmMtebEncoder(vllm_model), MTEB_EMBED_TASKS
|
||||
VllmMtebEncoder(vllm_model, prompt_prefix=prompt_prefix), MTEB_EMBED_TASKS
|
||||
)
|
||||
vllm_dtype = vllm_model.llm.llm_engine.model_config.dtype
|
||||
head_dtype = model_config.head_dtype
|
||||
@@ -210,7 +230,9 @@ def mteb_test_embed_models(
|
||||
if hf_model_callback is not None:
|
||||
hf_model_callback(hf_model)
|
||||
|
||||
st_main_score = run_mteb_embed_task(hf_model, MTEB_EMBED_TASKS)
|
||||
st_main_score = run_mteb_embed_task(
|
||||
HfMtebEncoder(hf_model), MTEB_EMBED_TASKS
|
||||
)
|
||||
st_dtype = next(hf_model.model.parameters()).dtype
|
||||
|
||||
# Check embeddings close to hf outputs
|
||||
|
||||
@@ -28,7 +28,16 @@ EMBEDDING_MODELS = [
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
)
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"jinaai/jina-embeddings-v5-text-small",
|
||||
mteb_score=0.794535707854956,
|
||||
architecture="JinaEmbeddingsV5Model",
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
),
|
||||
]
|
||||
|
||||
RERANK_MODELS = [
|
||||
@@ -46,11 +55,18 @@ RERANK_MODELS = [
|
||||
|
||||
@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
task = "retrieval" if "v5" in model_info.name else "text-matching"
|
||||
prompt_prefix: str | None = "Document: " if "v5" in model_info.name else None
|
||||
|
||||
def hf_model_callback(model):
|
||||
model.encode = partial(model.encode, task="text-matching")
|
||||
model.encode = partial(model.encode, task=task)
|
||||
|
||||
mteb_test_embed_models(
|
||||
hf_runner, vllm_runner, model_info, hf_model_callback=hf_model_callback
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
hf_model_callback=hf_model_callback,
|
||||
prompt_prefix=prompt_prefix,
|
||||
)
|
||||
|
||||
|
||||
@@ -58,8 +74,10 @@ def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -
|
||||
def test_embed_models_correctness(
|
||||
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
|
||||
) -> None:
|
||||
task = "retrieval" if "v5" in model_info.name else "text-matching"
|
||||
|
||||
def hf_model_callback(model):
|
||||
model.encode = partial(model.encode, task="text-matching")
|
||||
model.encode = partial(model.encode, task=task)
|
||||
|
||||
correctness_test_embed_models(
|
||||
hf_runner,
|
||||
@@ -97,12 +115,14 @@ def test_matryoshka(
|
||||
# ST will strip the input texts, see test_embedding.py
|
||||
example_prompts = [str(s).strip() for s in example_prompts]
|
||||
|
||||
task = "retrieval" if "v5" in model_info.name else "text-matching"
|
||||
|
||||
with hf_runner(
|
||||
model_info.name,
|
||||
dtype=dtype,
|
||||
is_sentence_transformer=True,
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.encode(example_prompts, task="text-matching")
|
||||
hf_outputs = hf_model.encode(example_prompts, task=task)
|
||||
hf_outputs = matryoshka_fy(hf_outputs, dimensions)
|
||||
|
||||
with vllm_runner(
|
||||
|
||||
@@ -609,6 +609,10 @@ _EMBEDDING_EXAMPLE_MODELS = {
|
||||
trust_remote_code=True,
|
||||
hf_overrides={"architectures": ["GteNewModel"]},
|
||||
),
|
||||
"JinaEmbeddingsV5Model": _HfExamplesInfo(
|
||||
"jinaai/jina-embeddings-v5-text-small",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"LlamaModel": _HfExamplesInfo("llama", is_available_online=False),
|
||||
"LlamaBidirectionalModel": _HfExamplesInfo(
|
||||
"nvidia/llama-nemotron-embed-1b-v2", trust_remote_code=True
|
||||
|
||||
Reference in New Issue
Block a user