[Model] Support jina-embeddings-v5-text-nano (EuroBERT encoder backbone) (#50688)

Signed-off-by: Omkar Shewale <omkarshewale2001@gmail.com>
This commit is contained in:
Omkar Shewale
2026-08-03 01:37:08 +00:00
committed by GitHub
parent 0033211c0b
commit 9a4fd57cac
6 changed files with 153 additions and 89 deletions
+4 -2
View File
@@ -45,7 +45,7 @@ You can compute pairwise similarity scores to build a similarity matrix using th
| `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ |
| `GteModel` | Arctic-Embed-2.0-M | `Snowflake/snowflake-arctic-embed-m-v2.0`. | | |
| `GteNewModel` | mGTE-TRM (see note) | `Alibaba-NLP/gte-multilingual-base`, etc. | | |
| `JinaEmbeddingsV5Model`<sup>C</sup> | Qwen3-based with task-specific LoRA adapters | `jinaai/jina-embeddings-v5-text-small` (see note) | ✅︎ | ✅︎ |
| `JinaEmbeddingsV5Model`<sup>C</sup> | Qwen3-decoder or EuroBERT-encoder backbone with task-specific LoRA adapters | `jinaai/jina-embeddings-v5-text-small`, `jinaai/jina-embeddings-v5-text-nano` (see note) | ✅︎ | ✅︎ |
| `LlamaBidirectionalModel`<sup>C</sup> | Llama-based with bidirectional attention | `nvidia/llama-nemotron-embed-1b-v2`, etc. | ✅︎ | ✅︎ |
| `LlamaModel`<sup>C</sup>, `LlamaForCausalLM`<sup>C</sup>, `MistralModel`<sup>C</sup>, etc. | Llama-based | `intfloat/e5-mistral-7b-instruct`, etc. | ✅︎ | ✅︎ |
| `ModernBertModel` | ModernBERT-based | `Alibaba-NLP/gte-modernbert-base`, etc. | | |
@@ -75,7 +75,9 @@ You can compute pairwise similarity scores to build a similarity matrix using th
`jinaai/jina-embeddings-v3` supports multiple tasks through LoRA, while vllm temporarily only supports text-matching tasks by merging LoRA weights.
!!! note
`jinaai/jina-embeddings-v5-text-small` ships with four task-specific LoRA adapters
`jinaai/jina-embeddings-v5-text-small` (Qwen3 decoder) and
`jinaai/jina-embeddings-v5-text-nano` (bidirectional EuroBERT encoder,
`is_decoder=false`) ship with four task-specific LoRA adapters
(`retrieval`, `text-matching`, `classification`, `clustering`). vLLM merges the
selected adapter into the base weights at load time. Choose the task with
`--hf-overrides '{"jina_task": "<task>"}'`; the default is `retrieval`.
@@ -1,12 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Backbone validation for Jina Embeddings V5 (issue #50337).
"""Backbone validation for Jina Embeddings V5.
The V5 family ships two backbones under one `architectures` entry: `-small` is
a Qwen3 decoder (which vLLM implements), while `-nano` is a bidirectional
EuroBERT encoder (which it does not). Upstream ships a separate
`configuration_*.py` per repository, so the only signal distinguishing them is
`is_decoder`, which the encoder variant sets to False.
a Qwen3 decoder, while `-nano` is a bidirectional EuroBERT encoder. Upstream
ships a separate `configuration_*.py` per repository, so the only signal
distinguishing them is `is_decoder`, which the encoder variant sets to False.
`JinaEmbeddingsV5ModelConfig` uses it to enable bidirectional attention for the
encoder variant; `JinaEmbeddingsV5Model` then dispatches to the correct backbone.
"""
from types import SimpleNamespace
@@ -34,16 +35,18 @@ def test_registered_for_the_architecture():
@pytest.mark.cpu_test
def test_encoder_backbone_is_rejected():
"""An encoder checkpoint must fail fast, naming the reason.
def test_encoder_backbone_enables_bidirectional_attention():
"""An encoder checkpoint (is_decoder=False) is supported.
Without this, loading proceeds until every `q_norm`/`k_norm` weight is
reported as uninitialized, which does not point at the actual problem.
The handler sets is_causal=False so the Llama backbone uses
EncoderOnlyAttention; JinaEmbeddingsV5Model then dispatches to the encoder
implementation.
"""
model_config = _model_config(PretrainedConfig(is_decoder=False))
hf_config = PretrainedConfig(is_decoder=False)
with pytest.raises(NotImplementedError, match="is_decoder=False"):
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(model_config)
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(_model_config(hf_config))
assert hf_config.is_causal is False
@pytest.mark.cpu_test
@@ -207,9 +207,18 @@ def mteb_test_embed_models(
vllm_dtype = vllm_model.llm.llm_engine.model_config.dtype
head_dtype = model_config.head_dtype
# Test embedding_size, isnan and whether to use normalize
# Test embedding_size, isnan and whether to use normalize.
# Apply the same prompt_prefix used for scoring so this check compares
# identical effective inputs: HF's SentenceTransformer.encode() applies
# the model's default prompt (e.g. "Document: "), so vLLM must receive it
# too. This mirrors VllmMtebEncoder and changes no production behavior.
consistency_prompts = (
[prompt_prefix + p for p in example_prompts]
if prompt_prefix
else example_prompts
)
vllm_outputs = vllm_model.embed(
example_prompts,
consistency_prompts,
tokenization_kwargs=dict(truncate_prompt_tokens=-1),
)
outputs_tensor = torch.tensor(vllm_outputs)
@@ -38,6 +38,14 @@ EMBEDDING_MODELS = [
is_prefix_caching_supported=True,
is_chunked_prefill_supported=True,
),
EmbedModelInfo(
"jinaai/jina-embeddings-v5-text-nano",
architecture="JinaEmbeddingsV5Model",
seq_pooling_type="LAST",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
),
]
RERANK_MODELS = [
+5 -14
View File
@@ -479,30 +479,21 @@ class JinaEmbeddingsV5ModelConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
"""Reject checkpoints whose backbone is not the Qwen3 decoder.
"""Enable the bidirectional encoder backbone for -nano checkpoints.
The V5 family ships more than one backbone under a single
`architectures` entry: the `-small` variants are Qwen3 decoders, while
`-nano` is a bidirectional EuroBERT encoder. Upstream ships a separate
`configuration_*.py` per repository, so the config carries no backbone
field and the encoder variants are only identifiable by
`is_decoder=False`. `JinaEmbeddingsV5Model` extends `Qwen3ForCausalLM`
and has no path to an encoder backbone, so without this check such a
checkpoint fails later with every `q_norm` and `k_norm` weight reported
as uninitialized, which does not point at the actual problem.
Raises:
NotImplementedError: If the checkpoint uses an encoder backbone.
`is_decoder=False`. For encoder checkpoints, set `is_causal=False` so the
Llama backbone uses EncoderOnlyAttention; `JinaEmbeddingsV5Model` then
dispatches to its encoder implementation.
"""
if getattr(model_config.hf_config, "is_decoder", True):
return
raise NotImplementedError(
"This jina-embeddings-v5 checkpoint uses a bidirectional encoder "
"backbone (is_decoder=False). JinaEmbeddingsV5Model is built on "
"Qwen3 and cannot serve it, so only the Qwen3-based decoder "
"variants are supported, such as jina-embeddings-v5-text-small."
)
model_config.hf_config.is_causal = False
class JinaForRankingConfig(VerifyAndUpdateConfig):
+110 -59
View File
@@ -24,6 +24,7 @@ from ..layers.pooler.tokwise import (
)
from .interfaces import SupportsLateInteraction
from .interfaces_base import VllmModelForPooling
from .llama import LlamaForCausalLM
from .qwen3 import Qwen3ForCausalLM, Qwen3Model
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
@@ -185,11 +186,79 @@ def _build_lora_pairs(adapter_weights: dict) -> dict:
return dict(lora_pairs)
class JinaEmbeddingsV5Model(Qwen3ForCausalLM, VllmModelForPooling):
"""Jina Embeddings V5 with task-specific LoRA adapters merged at load time.
def _setup_jina_v5_task_and_pooler(model: nn.Module, vllm_config: VllmConfig) -> None:
"""Shared init for jina-embeddings-v5 wrappers: select task + build pooler."""
model._model_name = vllm_config.model_config.model
model._revision = vllm_config.model_config.revision
Extends Qwen3ForCausalLM (the underlying architecture) and declares itself
as a pooling model so that as_embedding_model() does not wrap it.
model._task = getattr(
vllm_config.model_config.hf_config, "jina_task", _DEFAULT_TASK
)
if model._task not in _SUPPORTED_TASKS:
logger.warning(
"Unknown jina_task=%r. Falling back to %r.",
model._task,
_DEFAULT_TASK,
)
model._task = _DEFAULT_TASK
pooler_config = vllm_config.model_config.pooler_config
assert pooler_config is not None
model.pooler = DispatchPooler.for_embedding(pooler_config)
def _load_jina_v5_weights(
model: nn.Module, weights: Iterable[tuple[str, torch.Tensor]]
) -> set[str]:
"""Shared loader: merge the selected task LoRA adapter into the base weights."""
lora_pairs: dict = {}
scaling = 1.0
result = _load_adapter(model._model_name, model._task, model._revision)
if result is None:
logger.warning(
"No adapter found for task %r in %r. Loading raw base weights.",
model._task,
model._model_name,
)
else:
adapter_config, adapter_weights = result
scaling = adapter_config["lora_alpha"] / adapter_config["r"]
lora_pairs = _build_lora_pairs(adapter_weights)
logger.info(
"Loaded %d adapter tensors for task %r (scaling=%.4f, %d LoRA pairs)",
len(adapter_weights),
model._task,
scaling,
len(lora_pairs),
)
def _merge_weights(
weights: Iterable[tuple[str, torch.Tensor]],
) -> Iterable[tuple[str, torch.Tensor]]:
for name, tensor in weights:
clean_name = name
if clean_name.startswith("model."):
clean_name = clean_name[len("model.") :]
if clean_name in lora_pairs:
pair = lora_pairs[clean_name]
if "A" in pair and "B" in pair:
lora_A = pair["A"].to(device=tensor.device, dtype=tensor.dtype)
lora_B = pair["B"].to(device=tensor.device, dtype=tensor.dtype)
tensor = tensor + (lora_B @ lora_A) * scaling
yield name, tensor
loader = AutoWeightsLoader(model, ignore_unexpected_prefixes=["lm_head."])
weights = _merge_weights(weights)
return loader.load_weights(weights, mapper=model.hf_to_vllm_mapper)
class JinaEmbeddingsV5DecoderModel(Qwen3ForCausalLM, VllmModelForPooling):
"""jina-embeddings-v5 with a Qwen3 decoder backbone (e.g. -small).
Task-specific LoRA adapters are merged into the base weights at load time.
Declares itself a pooling model so that as_embedding_model() does not wrap it.
"""
is_pooling_model = True
@@ -199,64 +268,46 @@ class JinaEmbeddingsV5Model(Qwen3ForCausalLM, VllmModelForPooling):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
self._model_name = vllm_config.model_config.model
self._revision = vllm_config.model_config.revision
self._task = getattr(
vllm_config.model_config.hf_config, "jina_task", _DEFAULT_TASK
)
if self._task not in _SUPPORTED_TASKS:
logger.warning(
"Unknown jina_task=%r. Falling back to %r.",
self._task,
_DEFAULT_TASK,
)
self._task = _DEFAULT_TASK
pooler_config = vllm_config.model_config.pooler_config
assert pooler_config is not None
self.pooler = DispatchPooler.for_embedding(pooler_config)
_setup_jina_v5_task_and_pooler(self, vllm_config)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
lora_pairs: dict = {}
scaling = 1.0
return _load_jina_v5_weights(self, weights)
result = _load_adapter(self._model_name, self._task, self._revision)
if result is None:
logger.warning(
"No adapter found for task %r in %r. Loading raw base weights.",
self._task,
self._model_name,
)
else:
adapter_config, adapter_weights = result
scaling = adapter_config["lora_alpha"] / adapter_config["r"]
lora_pairs = _build_lora_pairs(adapter_weights)
logger.info(
"Loaded %d adapter tensors for task %r (scaling=%.4f, %d LoRA pairs)",
len(adapter_weights),
self._task,
scaling,
len(lora_pairs),
)
def _merge_weights(
weights: Iterable[tuple[str, torch.Tensor]],
) -> Iterable[tuple[str, torch.Tensor]]:
for name, tensor in weights:
clean_name = name
if clean_name.startswith("model."):
clean_name = clean_name[len("model.") :]
class JinaEmbeddingsV5EncoderModel(LlamaForCausalLM, VllmModelForPooling):
"""jina-embeddings-v5 with a bidirectional EuroBERT (Llama) encoder backbone.
if clean_name in lora_pairs:
pair = lora_pairs[clean_name]
if "A" in pair and "B" in pair:
lora_A = pair["A"].to(device=tensor.device, dtype=tensor.dtype)
lora_B = pair["B"].to(device=tensor.device, dtype=tensor.dtype)
tensor = tensor + (lora_B @ lora_A) * scaling
yield name, tensor
Used by encoder checkpoints such as jina-embeddings-v5-text-nano
(``is_decoder=False``). EuroBERT is architecturally a bidirectional Llama, so
the LlamaModel backbone switches to EncoderOnlyAttention when the config
carries ``is_causal=False`` (set by ``JinaEmbeddingsV5ModelConfig``).
"""
loader = AutoWeightsLoader(self, ignore_unexpected_prefixes=["lm_head."])
weights = _merge_weights(weights)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
is_pooling_model = True
hf_to_vllm_mapper = LlamaForCausalLM.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_prefix={"": "model."}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
_setup_jina_v5_task_and_pooler(self, vllm_config)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
return _load_jina_v5_weights(self, weights)
class JinaEmbeddingsV5Model(JinaEmbeddingsV5DecoderModel):
"""Dispatcher for the jina-embeddings-v5 family.
The family ships two backbones under one ``architectures`` entry: Qwen3
decoders (-small) and bidirectional EuroBERT encoders (-nano), told apart by
``is_decoder``. Inherits the decoder implementation so registry introspection
still sees a valid pooling model, and ``__new__`` swaps in the encoder
variant for encoder checkpoints.
"""
def __new__(cls, *, vllm_config: VllmConfig, prefix: str = ""):
is_decoder = getattr(vllm_config.model_config.hf_config, "is_decoder", True)
if not is_decoder:
return JinaEmbeddingsV5EncoderModel(vllm_config=vllm_config, prefix=prefix)
return super().__new__(cls)