From 9a4fd57cac19b334b420ae24226c1d2151566799 Mon Sep 17 00:00:00 2001 From: Omkar Shewale <84273555+omkar-droid@users.noreply.github.com> Date: Sun, 2 Aug 2026 20:37:08 -0500 Subject: [PATCH] [Model] Support jina-embeddings-v5-text-nano (EuroBERT encoder backbone) (#50688) Signed-off-by: Omkar Shewale --- docs/models/pooling_models/embed.md | 6 +- .../pooling/test_jina_embeddings_v5.py | 27 +-- .../pooling_mteb_test/mteb_embed_utils.py | 13 +- .../language/pooling_mteb_test/test_jina.py | 8 + vllm/model_executor/models/config.py | 19 +- vllm/model_executor/models/jina.py | 169 ++++++++++++------ 6 files changed, 153 insertions(+), 89 deletions(-) diff --git a/docs/models/pooling_models/embed.md b/docs/models/pooling_models/embed.md index b680f06401..c548ed80d2 100644 --- a/docs/models/pooling_models/embed.md +++ b/docs/models/pooling_models/embed.md @@ -45,7 +45,7 @@ You can compute pairwise similarity scores to build a similarity matrix using th | `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ | | `GteModel` | Arctic-Embed-2.0-M | `Snowflake/snowflake-arctic-embed-m-v2.0`. | | | | `GteNewModel` | mGTE-TRM (see note) | `Alibaba-NLP/gte-multilingual-base`, etc. | | | -| `JinaEmbeddingsV5Model`C | Qwen3-based with task-specific LoRA adapters | `jinaai/jina-embeddings-v5-text-small` (see note) | ✅︎ | ✅︎ | +| `JinaEmbeddingsV5Model`C | Qwen3-decoder or EuroBERT-encoder backbone with task-specific LoRA adapters | `jinaai/jina-embeddings-v5-text-small`, `jinaai/jina-embeddings-v5-text-nano` (see note) | ✅︎ | ✅︎ | | `LlamaBidirectionalModel`C | Llama-based with bidirectional attention | `nvidia/llama-nemotron-embed-1b-v2`, etc. | ✅︎ | ✅︎ | | `LlamaModel`C, `LlamaForCausalLM`C, `MistralModel`C, etc. | Llama-based | `intfloat/e5-mistral-7b-instruct`, etc. | ✅︎ | ✅︎ | | `ModernBertModel` | ModernBERT-based | `Alibaba-NLP/gte-modernbert-base`, etc. | | | @@ -75,7 +75,9 @@ You can compute pairwise similarity scores to build a similarity matrix using th `jinaai/jina-embeddings-v3` supports multiple tasks through LoRA, while vllm temporarily only supports text-matching tasks by merging LoRA weights. !!! note - `jinaai/jina-embeddings-v5-text-small` ships with four task-specific LoRA adapters + `jinaai/jina-embeddings-v5-text-small` (Qwen3 decoder) and + `jinaai/jina-embeddings-v5-text-nano` (bidirectional EuroBERT encoder, + `is_decoder=false`) ship with four task-specific LoRA adapters (`retrieval`, `text-matching`, `classification`, `clustering`). vLLM merges the selected adapter into the base weights at load time. Choose the task with `--hf-overrides '{"jina_task": ""}'`; the default is `retrieval`. diff --git a/tests/models/language/pooling/test_jina_embeddings_v5.py b/tests/models/language/pooling/test_jina_embeddings_v5.py index d6a9243d17..fbfebcc5ce 100644 --- a/tests/models/language/pooling/test_jina_embeddings_v5.py +++ b/tests/models/language/pooling/test_jina_embeddings_v5.py @@ -1,12 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Backbone validation for Jina Embeddings V5 (issue #50337). +"""Backbone validation for Jina Embeddings V5. The V5 family ships two backbones under one `architectures` entry: `-small` is -a Qwen3 decoder (which vLLM implements), while `-nano` is a bidirectional -EuroBERT encoder (which it does not). Upstream ships a separate -`configuration_*.py` per repository, so the only signal distinguishing them is -`is_decoder`, which the encoder variant sets to False. +a Qwen3 decoder, while `-nano` is a bidirectional EuroBERT encoder. Upstream +ships a separate `configuration_*.py` per repository, so the only signal +distinguishing them is `is_decoder`, which the encoder variant sets to False. +`JinaEmbeddingsV5ModelConfig` uses it to enable bidirectional attention for the +encoder variant; `JinaEmbeddingsV5Model` then dispatches to the correct backbone. """ from types import SimpleNamespace @@ -34,16 +35,18 @@ def test_registered_for_the_architecture(): @pytest.mark.cpu_test -def test_encoder_backbone_is_rejected(): - """An encoder checkpoint must fail fast, naming the reason. +def test_encoder_backbone_enables_bidirectional_attention(): + """An encoder checkpoint (is_decoder=False) is supported. - Without this, loading proceeds until every `q_norm`/`k_norm` weight is - reported as uninitialized, which does not point at the actual problem. + The handler sets is_causal=False so the Llama backbone uses + EncoderOnlyAttention; JinaEmbeddingsV5Model then dispatches to the encoder + implementation. """ - model_config = _model_config(PretrainedConfig(is_decoder=False)) + hf_config = PretrainedConfig(is_decoder=False) - with pytest.raises(NotImplementedError, match="is_decoder=False"): - JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(model_config) + JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(_model_config(hf_config)) + + assert hf_config.is_causal is False @pytest.mark.cpu_test diff --git a/tests/models/language/pooling_mteb_test/mteb_embed_utils.py b/tests/models/language/pooling_mteb_test/mteb_embed_utils.py index fc575c399d..101a0a4690 100644 --- a/tests/models/language/pooling_mteb_test/mteb_embed_utils.py +++ b/tests/models/language/pooling_mteb_test/mteb_embed_utils.py @@ -207,9 +207,18 @@ def mteb_test_embed_models( vllm_dtype = vllm_model.llm.llm_engine.model_config.dtype head_dtype = model_config.head_dtype - # Test embedding_size, isnan and whether to use normalize + # Test embedding_size, isnan and whether to use normalize. + # Apply the same prompt_prefix used for scoring so this check compares + # identical effective inputs: HF's SentenceTransformer.encode() applies + # the model's default prompt (e.g. "Document: "), so vLLM must receive it + # too. This mirrors VllmMtebEncoder and changes no production behavior. + consistency_prompts = ( + [prompt_prefix + p for p in example_prompts] + if prompt_prefix + else example_prompts + ) vllm_outputs = vllm_model.embed( - example_prompts, + consistency_prompts, tokenization_kwargs=dict(truncate_prompt_tokens=-1), ) outputs_tensor = torch.tensor(vllm_outputs) diff --git a/tests/models/language/pooling_mteb_test/test_jina.py b/tests/models/language/pooling_mteb_test/test_jina.py index 24aa3188f8..fd6fd96f14 100644 --- a/tests/models/language/pooling_mteb_test/test_jina.py +++ b/tests/models/language/pooling_mteb_test/test_jina.py @@ -38,6 +38,14 @@ EMBEDDING_MODELS = [ is_prefix_caching_supported=True, is_chunked_prefill_supported=True, ), + EmbedModelInfo( + "jinaai/jina-embeddings-v5-text-nano", + architecture="JinaEmbeddingsV5Model", + seq_pooling_type="LAST", + attn_type="encoder_only", + is_prefix_caching_supported=False, + is_chunked_prefill_supported=False, + ), ] RERANK_MODELS = [ diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index cea07b6e85..66427dfcdd 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -479,30 +479,21 @@ class JinaEmbeddingsV5ModelConfig(VerifyAndUpdateConfig): @staticmethod def verify_and_update_model_config(model_config: "ModelConfig") -> None: - """Reject checkpoints whose backbone is not the Qwen3 decoder. + """Enable the bidirectional encoder backbone for -nano checkpoints. The V5 family ships more than one backbone under a single `architectures` entry: the `-small` variants are Qwen3 decoders, while `-nano` is a bidirectional EuroBERT encoder. Upstream ships a separate `configuration_*.py` per repository, so the config carries no backbone field and the encoder variants are only identifiable by - `is_decoder=False`. `JinaEmbeddingsV5Model` extends `Qwen3ForCausalLM` - and has no path to an encoder backbone, so without this check such a - checkpoint fails later with every `q_norm` and `k_norm` weight reported - as uninitialized, which does not point at the actual problem. - - Raises: - NotImplementedError: If the checkpoint uses an encoder backbone. + `is_decoder=False`. For encoder checkpoints, set `is_causal=False` so the + Llama backbone uses EncoderOnlyAttention; `JinaEmbeddingsV5Model` then + dispatches to its encoder implementation. """ if getattr(model_config.hf_config, "is_decoder", True): return - raise NotImplementedError( - "This jina-embeddings-v5 checkpoint uses a bidirectional encoder " - "backbone (is_decoder=False). JinaEmbeddingsV5Model is built on " - "Qwen3 and cannot serve it, so only the Qwen3-based decoder " - "variants are supported, such as jina-embeddings-v5-text-small." - ) + model_config.hf_config.is_causal = False class JinaForRankingConfig(VerifyAndUpdateConfig): diff --git a/vllm/model_executor/models/jina.py b/vllm/model_executor/models/jina.py index 06f5ce282c..63d690bb3d 100644 --- a/vllm/model_executor/models/jina.py +++ b/vllm/model_executor/models/jina.py @@ -24,6 +24,7 @@ from ..layers.pooler.tokwise import ( ) from .interfaces import SupportsLateInteraction from .interfaces_base import VllmModelForPooling +from .llama import LlamaForCausalLM from .qwen3 import Qwen3ForCausalLM, Qwen3Model from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix @@ -185,11 +186,79 @@ def _build_lora_pairs(adapter_weights: dict) -> dict: return dict(lora_pairs) -class JinaEmbeddingsV5Model(Qwen3ForCausalLM, VllmModelForPooling): - """Jina Embeddings V5 with task-specific LoRA adapters merged at load time. +def _setup_jina_v5_task_and_pooler(model: nn.Module, vllm_config: VllmConfig) -> None: + """Shared init for jina-embeddings-v5 wrappers: select task + build pooler.""" + model._model_name = vllm_config.model_config.model + model._revision = vllm_config.model_config.revision - Extends Qwen3ForCausalLM (the underlying architecture) and declares itself - as a pooling model so that as_embedding_model() does not wrap it. + model._task = getattr( + vllm_config.model_config.hf_config, "jina_task", _DEFAULT_TASK + ) + if model._task not in _SUPPORTED_TASKS: + logger.warning( + "Unknown jina_task=%r. Falling back to %r.", + model._task, + _DEFAULT_TASK, + ) + model._task = _DEFAULT_TASK + + pooler_config = vllm_config.model_config.pooler_config + assert pooler_config is not None + model.pooler = DispatchPooler.for_embedding(pooler_config) + + +def _load_jina_v5_weights( + model: nn.Module, weights: Iterable[tuple[str, torch.Tensor]] +) -> set[str]: + """Shared loader: merge the selected task LoRA adapter into the base weights.""" + lora_pairs: dict = {} + scaling = 1.0 + + result = _load_adapter(model._model_name, model._task, model._revision) + if result is None: + logger.warning( + "No adapter found for task %r in %r. Loading raw base weights.", + model._task, + model._model_name, + ) + else: + adapter_config, adapter_weights = result + scaling = adapter_config["lora_alpha"] / adapter_config["r"] + lora_pairs = _build_lora_pairs(adapter_weights) + logger.info( + "Loaded %d adapter tensors for task %r (scaling=%.4f, %d LoRA pairs)", + len(adapter_weights), + model._task, + scaling, + len(lora_pairs), + ) + + def _merge_weights( + weights: Iterable[tuple[str, torch.Tensor]], + ) -> Iterable[tuple[str, torch.Tensor]]: + for name, tensor in weights: + clean_name = name + if clean_name.startswith("model."): + clean_name = clean_name[len("model.") :] + + if clean_name in lora_pairs: + pair = lora_pairs[clean_name] + if "A" in pair and "B" in pair: + lora_A = pair["A"].to(device=tensor.device, dtype=tensor.dtype) + lora_B = pair["B"].to(device=tensor.device, dtype=tensor.dtype) + tensor = tensor + (lora_B @ lora_A) * scaling + yield name, tensor + + loader = AutoWeightsLoader(model, ignore_unexpected_prefixes=["lm_head."]) + weights = _merge_weights(weights) + return loader.load_weights(weights, mapper=model.hf_to_vllm_mapper) + + +class JinaEmbeddingsV5DecoderModel(Qwen3ForCausalLM, VllmModelForPooling): + """jina-embeddings-v5 with a Qwen3 decoder backbone (e.g. -small). + + Task-specific LoRA adapters are merged into the base weights at load time. + Declares itself a pooling model so that as_embedding_model() does not wrap it. """ is_pooling_model = True @@ -199,64 +268,46 @@ class JinaEmbeddingsV5Model(Qwen3ForCausalLM, VllmModelForPooling): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__(vllm_config=vllm_config, prefix=prefix) - - self._model_name = vllm_config.model_config.model - self._revision = vllm_config.model_config.revision - - self._task = getattr( - vllm_config.model_config.hf_config, "jina_task", _DEFAULT_TASK - ) - if self._task not in _SUPPORTED_TASKS: - logger.warning( - "Unknown jina_task=%r. Falling back to %r.", - self._task, - _DEFAULT_TASK, - ) - self._task = _DEFAULT_TASK - - pooler_config = vllm_config.model_config.pooler_config - assert pooler_config is not None - self.pooler = DispatchPooler.for_embedding(pooler_config) + _setup_jina_v5_task_and_pooler(self, vllm_config) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - lora_pairs: dict = {} - scaling = 1.0 + return _load_jina_v5_weights(self, weights) - result = _load_adapter(self._model_name, self._task, self._revision) - if result is None: - logger.warning( - "No adapter found for task %r in %r. Loading raw base weights.", - self._task, - self._model_name, - ) - else: - adapter_config, adapter_weights = result - scaling = adapter_config["lora_alpha"] / adapter_config["r"] - lora_pairs = _build_lora_pairs(adapter_weights) - logger.info( - "Loaded %d adapter tensors for task %r (scaling=%.4f, %d LoRA pairs)", - len(adapter_weights), - self._task, - scaling, - len(lora_pairs), - ) - def _merge_weights( - weights: Iterable[tuple[str, torch.Tensor]], - ) -> Iterable[tuple[str, torch.Tensor]]: - for name, tensor in weights: - clean_name = name - if clean_name.startswith("model."): - clean_name = clean_name[len("model.") :] +class JinaEmbeddingsV5EncoderModel(LlamaForCausalLM, VllmModelForPooling): + """jina-embeddings-v5 with a bidirectional EuroBERT (Llama) encoder backbone. - if clean_name in lora_pairs: - pair = lora_pairs[clean_name] - if "A" in pair and "B" in pair: - lora_A = pair["A"].to(device=tensor.device, dtype=tensor.dtype) - lora_B = pair["B"].to(device=tensor.device, dtype=tensor.dtype) - tensor = tensor + (lora_B @ lora_A) * scaling - yield name, tensor + Used by encoder checkpoints such as jina-embeddings-v5-text-nano + (``is_decoder=False``). EuroBERT is architecturally a bidirectional Llama, so + the LlamaModel backbone switches to EncoderOnlyAttention when the config + carries ``is_causal=False`` (set by ``JinaEmbeddingsV5ModelConfig``). + """ - loader = AutoWeightsLoader(self, ignore_unexpected_prefixes=["lm_head."]) - weights = _merge_weights(weights) - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) + is_pooling_model = True + hf_to_vllm_mapper = LlamaForCausalLM.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_prefix={"": "model."} + ) + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__(vllm_config=vllm_config, prefix=prefix) + _setup_jina_v5_task_and_pooler(self, vllm_config) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + return _load_jina_v5_weights(self, weights) + + +class JinaEmbeddingsV5Model(JinaEmbeddingsV5DecoderModel): + """Dispatcher for the jina-embeddings-v5 family. + + The family ships two backbones under one ``architectures`` entry: Qwen3 + decoders (-small) and bidirectional EuroBERT encoders (-nano), told apart by + ``is_decoder``. Inherits the decoder implementation so registry introspection + still sees a valid pooling model, and ``__new__`` swaps in the encoder + variant for encoder checkpoints. + """ + + def __new__(cls, *, vllm_config: VllmConfig, prefix: str = ""): + is_decoder = getattr(vllm_config.model_config.hf_config, "is_decoder", True) + if not is_decoder: + return JinaEmbeddingsV5EncoderModel(vllm_config=vllm_config, prefix=prefix) + return super().__new__(cls)