[Model] Support jina-embeddings-v5-text-nano (EuroBERT encoder backbone) (#50688)
Signed-off-by: Omkar Shewale <omkarshewale2001@gmail.com>
This commit is contained in:
@@ -45,7 +45,7 @@ You can compute pairwise similarity scores to build a similarity matrix using th
|
||||
| `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ |
|
||||
| `GteModel` | Arctic-Embed-2.0-M | `Snowflake/snowflake-arctic-embed-m-v2.0`. | | |
|
||||
| `GteNewModel` | mGTE-TRM (see note) | `Alibaba-NLP/gte-multilingual-base`, etc. | | |
|
||||
| `JinaEmbeddingsV5Model`<sup>C</sup> | Qwen3-based with task-specific LoRA adapters | `jinaai/jina-embeddings-v5-text-small` (see note) | ✅︎ | ✅︎ |
|
||||
| `JinaEmbeddingsV5Model`<sup>C</sup> | Qwen3-decoder or EuroBERT-encoder backbone with task-specific LoRA adapters | `jinaai/jina-embeddings-v5-text-small`, `jinaai/jina-embeddings-v5-text-nano` (see note) | ✅︎ | ✅︎ |
|
||||
| `LlamaBidirectionalModel`<sup>C</sup> | Llama-based with bidirectional attention | `nvidia/llama-nemotron-embed-1b-v2`, etc. | ✅︎ | ✅︎ |
|
||||
| `LlamaModel`<sup>C</sup>, `LlamaForCausalLM`<sup>C</sup>, `MistralModel`<sup>C</sup>, etc. | Llama-based | `intfloat/e5-mistral-7b-instruct`, etc. | ✅︎ | ✅︎ |
|
||||
| `ModernBertModel` | ModernBERT-based | `Alibaba-NLP/gte-modernbert-base`, etc. | | |
|
||||
@@ -75,7 +75,9 @@ You can compute pairwise similarity scores to build a similarity matrix using th
|
||||
`jinaai/jina-embeddings-v3` supports multiple tasks through LoRA, while vllm temporarily only supports text-matching tasks by merging LoRA weights.
|
||||
|
||||
!!! note
|
||||
`jinaai/jina-embeddings-v5-text-small` ships with four task-specific LoRA adapters
|
||||
`jinaai/jina-embeddings-v5-text-small` (Qwen3 decoder) and
|
||||
`jinaai/jina-embeddings-v5-text-nano` (bidirectional EuroBERT encoder,
|
||||
`is_decoder=false`) ship with four task-specific LoRA adapters
|
||||
(`retrieval`, `text-matching`, `classification`, `clustering`). vLLM merges the
|
||||
selected adapter into the base weights at load time. Choose the task with
|
||||
`--hf-overrides '{"jina_task": "<task>"}'`; the default is `retrieval`.
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Backbone validation for Jina Embeddings V5 (issue #50337).
|
||||
"""Backbone validation for Jina Embeddings V5.
|
||||
|
||||
The V5 family ships two backbones under one `architectures` entry: `-small` is
|
||||
a Qwen3 decoder (which vLLM implements), while `-nano` is a bidirectional
|
||||
EuroBERT encoder (which it does not). Upstream ships a separate
|
||||
`configuration_*.py` per repository, so the only signal distinguishing them is
|
||||
`is_decoder`, which the encoder variant sets to False.
|
||||
a Qwen3 decoder, while `-nano` is a bidirectional EuroBERT encoder. Upstream
|
||||
ships a separate `configuration_*.py` per repository, so the only signal
|
||||
distinguishing them is `is_decoder`, which the encoder variant sets to False.
|
||||
`JinaEmbeddingsV5ModelConfig` uses it to enable bidirectional attention for the
|
||||
encoder variant; `JinaEmbeddingsV5Model` then dispatches to the correct backbone.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
@@ -34,16 +35,18 @@ def test_registered_for_the_architecture():
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_encoder_backbone_is_rejected():
|
||||
"""An encoder checkpoint must fail fast, naming the reason.
|
||||
def test_encoder_backbone_enables_bidirectional_attention():
|
||||
"""An encoder checkpoint (is_decoder=False) is supported.
|
||||
|
||||
Without this, loading proceeds until every `q_norm`/`k_norm` weight is
|
||||
reported as uninitialized, which does not point at the actual problem.
|
||||
The handler sets is_causal=False so the Llama backbone uses
|
||||
EncoderOnlyAttention; JinaEmbeddingsV5Model then dispatches to the encoder
|
||||
implementation.
|
||||
"""
|
||||
model_config = _model_config(PretrainedConfig(is_decoder=False))
|
||||
hf_config = PretrainedConfig(is_decoder=False)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="is_decoder=False"):
|
||||
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(model_config)
|
||||
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(_model_config(hf_config))
|
||||
|
||||
assert hf_config.is_causal is False
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
|
||||
@@ -207,9 +207,18 @@ def mteb_test_embed_models(
|
||||
vllm_dtype = vllm_model.llm.llm_engine.model_config.dtype
|
||||
head_dtype = model_config.head_dtype
|
||||
|
||||
# Test embedding_size, isnan and whether to use normalize
|
||||
# Test embedding_size, isnan and whether to use normalize.
|
||||
# Apply the same prompt_prefix used for scoring so this check compares
|
||||
# identical effective inputs: HF's SentenceTransformer.encode() applies
|
||||
# the model's default prompt (e.g. "Document: "), so vLLM must receive it
|
||||
# too. This mirrors VllmMtebEncoder and changes no production behavior.
|
||||
consistency_prompts = (
|
||||
[prompt_prefix + p for p in example_prompts]
|
||||
if prompt_prefix
|
||||
else example_prompts
|
||||
)
|
||||
vllm_outputs = vllm_model.embed(
|
||||
example_prompts,
|
||||
consistency_prompts,
|
||||
tokenization_kwargs=dict(truncate_prompt_tokens=-1),
|
||||
)
|
||||
outputs_tensor = torch.tensor(vllm_outputs)
|
||||
|
||||
@@ -38,6 +38,14 @@ EMBEDDING_MODELS = [
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"jinaai/jina-embeddings-v5-text-nano",
|
||||
architecture="JinaEmbeddingsV5Model",
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
),
|
||||
]
|
||||
|
||||
RERANK_MODELS = [
|
||||
|
||||
@@ -479,30 +479,21 @@ class JinaEmbeddingsV5ModelConfig(VerifyAndUpdateConfig):
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
"""Reject checkpoints whose backbone is not the Qwen3 decoder.
|
||||
"""Enable the bidirectional encoder backbone for -nano checkpoints.
|
||||
|
||||
The V5 family ships more than one backbone under a single
|
||||
`architectures` entry: the `-small` variants are Qwen3 decoders, while
|
||||
`-nano` is a bidirectional EuroBERT encoder. Upstream ships a separate
|
||||
`configuration_*.py` per repository, so the config carries no backbone
|
||||
field and the encoder variants are only identifiable by
|
||||
`is_decoder=False`. `JinaEmbeddingsV5Model` extends `Qwen3ForCausalLM`
|
||||
and has no path to an encoder backbone, so without this check such a
|
||||
checkpoint fails later with every `q_norm` and `k_norm` weight reported
|
||||
as uninitialized, which does not point at the actual problem.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: If the checkpoint uses an encoder backbone.
|
||||
`is_decoder=False`. For encoder checkpoints, set `is_causal=False` so the
|
||||
Llama backbone uses EncoderOnlyAttention; `JinaEmbeddingsV5Model` then
|
||||
dispatches to its encoder implementation.
|
||||
"""
|
||||
if getattr(model_config.hf_config, "is_decoder", True):
|
||||
return
|
||||
|
||||
raise NotImplementedError(
|
||||
"This jina-embeddings-v5 checkpoint uses a bidirectional encoder "
|
||||
"backbone (is_decoder=False). JinaEmbeddingsV5Model is built on "
|
||||
"Qwen3 and cannot serve it, so only the Qwen3-based decoder "
|
||||
"variants are supported, such as jina-embeddings-v5-text-small."
|
||||
)
|
||||
model_config.hf_config.is_causal = False
|
||||
|
||||
|
||||
class JinaForRankingConfig(VerifyAndUpdateConfig):
|
||||
|
||||
@@ -24,6 +24,7 @@ from ..layers.pooler.tokwise import (
|
||||
)
|
||||
from .interfaces import SupportsLateInteraction
|
||||
from .interfaces_base import VllmModelForPooling
|
||||
from .llama import LlamaForCausalLM
|
||||
from .qwen3 import Qwen3ForCausalLM, Qwen3Model
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
|
||||
@@ -185,11 +186,79 @@ def _build_lora_pairs(adapter_weights: dict) -> dict:
|
||||
return dict(lora_pairs)
|
||||
|
||||
|
||||
class JinaEmbeddingsV5Model(Qwen3ForCausalLM, VllmModelForPooling):
|
||||
"""Jina Embeddings V5 with task-specific LoRA adapters merged at load time.
|
||||
def _setup_jina_v5_task_and_pooler(model: nn.Module, vllm_config: VllmConfig) -> None:
|
||||
"""Shared init for jina-embeddings-v5 wrappers: select task + build pooler."""
|
||||
model._model_name = vllm_config.model_config.model
|
||||
model._revision = vllm_config.model_config.revision
|
||||
|
||||
Extends Qwen3ForCausalLM (the underlying architecture) and declares itself
|
||||
as a pooling model so that as_embedding_model() does not wrap it.
|
||||
model._task = getattr(
|
||||
vllm_config.model_config.hf_config, "jina_task", _DEFAULT_TASK
|
||||
)
|
||||
if model._task not in _SUPPORTED_TASKS:
|
||||
logger.warning(
|
||||
"Unknown jina_task=%r. Falling back to %r.",
|
||||
model._task,
|
||||
_DEFAULT_TASK,
|
||||
)
|
||||
model._task = _DEFAULT_TASK
|
||||
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
model.pooler = DispatchPooler.for_embedding(pooler_config)
|
||||
|
||||
|
||||
def _load_jina_v5_weights(
|
||||
model: nn.Module, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> set[str]:
|
||||
"""Shared loader: merge the selected task LoRA adapter into the base weights."""
|
||||
lora_pairs: dict = {}
|
||||
scaling = 1.0
|
||||
|
||||
result = _load_adapter(model._model_name, model._task, model._revision)
|
||||
if result is None:
|
||||
logger.warning(
|
||||
"No adapter found for task %r in %r. Loading raw base weights.",
|
||||
model._task,
|
||||
model._model_name,
|
||||
)
|
||||
else:
|
||||
adapter_config, adapter_weights = result
|
||||
scaling = adapter_config["lora_alpha"] / adapter_config["r"]
|
||||
lora_pairs = _build_lora_pairs(adapter_weights)
|
||||
logger.info(
|
||||
"Loaded %d adapter tensors for task %r (scaling=%.4f, %d LoRA pairs)",
|
||||
len(adapter_weights),
|
||||
model._task,
|
||||
scaling,
|
||||
len(lora_pairs),
|
||||
)
|
||||
|
||||
def _merge_weights(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
for name, tensor in weights:
|
||||
clean_name = name
|
||||
if clean_name.startswith("model."):
|
||||
clean_name = clean_name[len("model.") :]
|
||||
|
||||
if clean_name in lora_pairs:
|
||||
pair = lora_pairs[clean_name]
|
||||
if "A" in pair and "B" in pair:
|
||||
lora_A = pair["A"].to(device=tensor.device, dtype=tensor.dtype)
|
||||
lora_B = pair["B"].to(device=tensor.device, dtype=tensor.dtype)
|
||||
tensor = tensor + (lora_B @ lora_A) * scaling
|
||||
yield name, tensor
|
||||
|
||||
loader = AutoWeightsLoader(model, ignore_unexpected_prefixes=["lm_head."])
|
||||
weights = _merge_weights(weights)
|
||||
return loader.load_weights(weights, mapper=model.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class JinaEmbeddingsV5DecoderModel(Qwen3ForCausalLM, VllmModelForPooling):
|
||||
"""jina-embeddings-v5 with a Qwen3 decoder backbone (e.g. -small).
|
||||
|
||||
Task-specific LoRA adapters are merged into the base weights at load time.
|
||||
Declares itself a pooling model so that as_embedding_model() does not wrap it.
|
||||
"""
|
||||
|
||||
is_pooling_model = True
|
||||
@@ -199,64 +268,46 @@ class JinaEmbeddingsV5Model(Qwen3ForCausalLM, VllmModelForPooling):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
self._model_name = vllm_config.model_config.model
|
||||
self._revision = vllm_config.model_config.revision
|
||||
|
||||
self._task = getattr(
|
||||
vllm_config.model_config.hf_config, "jina_task", _DEFAULT_TASK
|
||||
)
|
||||
if self._task not in _SUPPORTED_TASKS:
|
||||
logger.warning(
|
||||
"Unknown jina_task=%r. Falling back to %r.",
|
||||
self._task,
|
||||
_DEFAULT_TASK,
|
||||
)
|
||||
self._task = _DEFAULT_TASK
|
||||
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
self.pooler = DispatchPooler.for_embedding(pooler_config)
|
||||
_setup_jina_v5_task_and_pooler(self, vllm_config)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
lora_pairs: dict = {}
|
||||
scaling = 1.0
|
||||
return _load_jina_v5_weights(self, weights)
|
||||
|
||||
result = _load_adapter(self._model_name, self._task, self._revision)
|
||||
if result is None:
|
||||
logger.warning(
|
||||
"No adapter found for task %r in %r. Loading raw base weights.",
|
||||
self._task,
|
||||
self._model_name,
|
||||
)
|
||||
else:
|
||||
adapter_config, adapter_weights = result
|
||||
scaling = adapter_config["lora_alpha"] / adapter_config["r"]
|
||||
lora_pairs = _build_lora_pairs(adapter_weights)
|
||||
logger.info(
|
||||
"Loaded %d adapter tensors for task %r (scaling=%.4f, %d LoRA pairs)",
|
||||
len(adapter_weights),
|
||||
self._task,
|
||||
scaling,
|
||||
len(lora_pairs),
|
||||
)
|
||||
|
||||
def _merge_weights(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
for name, tensor in weights:
|
||||
clean_name = name
|
||||
if clean_name.startswith("model."):
|
||||
clean_name = clean_name[len("model.") :]
|
||||
class JinaEmbeddingsV5EncoderModel(LlamaForCausalLM, VllmModelForPooling):
|
||||
"""jina-embeddings-v5 with a bidirectional EuroBERT (Llama) encoder backbone.
|
||||
|
||||
if clean_name in lora_pairs:
|
||||
pair = lora_pairs[clean_name]
|
||||
if "A" in pair and "B" in pair:
|
||||
lora_A = pair["A"].to(device=tensor.device, dtype=tensor.dtype)
|
||||
lora_B = pair["B"].to(device=tensor.device, dtype=tensor.dtype)
|
||||
tensor = tensor + (lora_B @ lora_A) * scaling
|
||||
yield name, tensor
|
||||
Used by encoder checkpoints such as jina-embeddings-v5-text-nano
|
||||
(``is_decoder=False``). EuroBERT is architecturally a bidirectional Llama, so
|
||||
the LlamaModel backbone switches to EncoderOnlyAttention when the config
|
||||
carries ``is_causal=False`` (set by ``JinaEmbeddingsV5ModelConfig``).
|
||||
"""
|
||||
|
||||
loader = AutoWeightsLoader(self, ignore_unexpected_prefixes=["lm_head."])
|
||||
weights = _merge_weights(weights)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
is_pooling_model = True
|
||||
hf_to_vllm_mapper = LlamaForCausalLM.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_prefix={"": "model."}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
_setup_jina_v5_task_and_pooler(self, vllm_config)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
return _load_jina_v5_weights(self, weights)
|
||||
|
||||
|
||||
class JinaEmbeddingsV5Model(JinaEmbeddingsV5DecoderModel):
|
||||
"""Dispatcher for the jina-embeddings-v5 family.
|
||||
|
||||
The family ships two backbones under one ``architectures`` entry: Qwen3
|
||||
decoders (-small) and bidirectional EuroBERT encoders (-nano), told apart by
|
||||
``is_decoder``. Inherits the decoder implementation so registry introspection
|
||||
still sees a valid pooling model, and ``__new__`` swaps in the encoder
|
||||
variant for encoder checkpoints.
|
||||
"""
|
||||
|
||||
def __new__(cls, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
is_decoder = getattr(vllm_config.model_config.hf_config, "is_decoder", True)
|
||||
if not is_decoder:
|
||||
return JinaEmbeddingsV5EncoderModel(vllm_config=vllm_config, prefix=prefix)
|
||||
return super().__new__(cls)
|
||||
|
||||
Reference in New Issue
Block a user