[New Model]: jinaai/jina-reranker-v3 (#38800)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-04-10 15:20:40 +00:00
committed by GitHub
parent 8d0f908b98
commit cb5f7501cb
14 changed files with 661 additions and 20 deletions
+52 -1
View File
@@ -24,7 +24,13 @@ from vllm.entrypoints.pooling.embed.protocol import (
EmbeddingChatRequest,
EmbeddingCompletionRequest,
)
from vllm.entrypoints.pooling.typing import PoolingServeContext
from vllm.entrypoints.pooling.scoring.io_processor import JinaRankingIOProcessorMixin
from vllm.entrypoints.pooling.typing import (
OfflineInputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
PoolingServeContext,
)
from vllm.inputs import EngineInput, tokens_input
from vllm.logger import init_logger
from vllm.outputs import PoolingOutput, PoolingRequestOutput
@@ -553,3 +559,48 @@ class EmbedIOProcessor(PoolingIOProcessor):
class TokenEmbedIOProcessor(PoolingIOProcessor):
name = "token_embed"
class JinaRankingTokenEmbedIOProcessor(
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
):
def pre_process_online(self, ctx: PoolingServeContext):
request = ctx.request
if isinstance(request, PoolingCompletionLikeRequest):
prompts = request.input
if not isinstance(prompts, Sequence) or len(prompts) < 2:
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
text_prompts = self.ensure_str(prompts)
# The JinaForRanking model concatenates docs first, then query.
# Let's stay consistent with this novel design.
prompt_input = self.format_docs_prompts_func(
query=text_prompts[-1], docs=text_prompts[:-1]
)
engine_inputs = self._preprocess_completion_online(
request,
prompt_input=prompt_input,
prompt_embeds=None,
)
elif isinstance(request, PoolingChatLikeRequest):
raise ValueError("The JinaForRanking does not support chat Request.")
else:
raise ValueError(f"Invalid {self.name} request type")
ctx.engine_inputs = engine_inputs
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2:
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
text_prompts = self.ensure_str(ctx.prompts)
# The JinaForRanking model concatenates docs first, then query.
# Let's stay consistent with this novel design.
ctx.prompts = self.format_docs_prompts_func(
query=text_prompts[-1], docs=text_prompts[:-1]
)
return super().pre_process_offline(ctx)
@@ -59,6 +59,13 @@ def init_pooling_io_processors(
if score_type is not None and score_type in ScoringIOProcessors:
processors[score_type] = ScoringIOProcessors[score_type]
if model_config.architecture == "JinaForRanking":
from .embed.io_processor import JinaRankingTokenEmbedIOProcessor
from .scoring.io_processor import ScoringIOProcessors
processors["token_embed"] = JinaRankingTokenEmbedIOProcessor
processors["late-interaction"] = ScoringIOProcessors["jina-reranking-scoring"]
return {
task: processor_cls(
vllm_config=vllm_config,
@@ -416,11 +416,137 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
return full_prompt, engine_prompt
class JinaRankingIOProcessorMixin:
@staticmethod
def sanitize_input(text: str, special_tokens: dict[str, str]) -> str:
for token in special_tokens.values():
text = text.replace(token, "")
return text
@staticmethod
def format_docs_prompts_func(
query: str,
docs: list[str],
special_tokens: dict[str, str] | None = None,
instruction: str | None = None,
no_thinking: bool = True,
) -> str:
# TODO: Try converting the code below into a chat template.
default_special_tokens = {
"query_embed_token": "<|rerank_token|>",
"doc_embed_token": "<|embed_token|>",
}
if special_tokens is None:
special_tokens = default_special_tokens
query = JinaRankingIOProcessorMixin.sanitize_input(query, special_tokens)
docs = [
JinaRankingIOProcessorMixin.sanitize_input(doc, special_tokens)
for doc in docs
]
prefix = (
"<|im_start|>system\n"
"You are a search relevance expert who can determine a ranking of the passages based on how relevant they are to the query. " # noqa: E501
"If the query is a question, how relevant a passage is depends on how well it answers the question. " # noqa: E501
"If not, try to analyze the intent of the query and assess how well each passage satisfies the intent. " # noqa: E501
"If an instruction is provided, you should follow the instruction when determining the ranking." # noqa: E501
"<|im_end|>\n<|im_start|>user\n"
)
suffix = "<|im_end|>\n<|im_start|>assistant\n"
if no_thinking:
suffix += "<think>\n\n</think>\n\n"
doc_emb_token = special_tokens["doc_embed_token"]
query_emb_token = special_tokens["query_embed_token"]
prompt = (
f"I will provide you with {len(docs)} passages, each indicated by a numerical identifier. " # noqa: E501
f"Rank the passages based on their relevance to query: {query}\n"
)
if instruction:
prompt += f"<instruct>\n{instruction}\n</instruct>\n"
doc_prompts = [
f'<passage id="{i}">\n{doc}{doc_emb_token}\n</passage>'
for i, doc in enumerate(docs)
]
prompt += "\n".join(doc_prompts) + "\n"
prompt += f"<query>\n{query}{query_emb_token}\n</query>"
return prefix + prompt + suffix
@staticmethod
def ensure_str(data: Sequence[Any]) -> list[str]:
text: list[str] = []
for prompt in data:
if not isinstance(prompt, str):
raise ValueError(
"The JinaForRanking model only supports text as input."
)
text.append(prompt)
return text
class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorMixin):
name = "jina-reranking-scoring"
pooling_task: PoolingTask = "token_embed"
def _pre_process(
self,
scoring_data: ScoringData,
tok_params: TokenizeParams,
prompt_extras: dict[str, Any] | None = None,
) -> Sequence[EngineInput]:
queries = self.ensure_str(scoring_data.data_1)
docs = self.ensure_str(scoring_data.data_2)
if len(queries) == 1:
prompts = [self.format_docs_prompts_func(query=queries[0], docs=docs)]
else:
prompts = [
self.format_docs_prompts_func(query=q, docs=[d])
for q, d in zip(queries, docs)
]
return self._preprocess_completion_offline(
prompts=prompts, tok_params=tok_params, prompt_extras=prompt_extras
)
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
final_res_batch: list[PoolingRequestOutput] = []
for i in range(len(outputs)):
embeds = outputs[i].outputs.data.float()
# The JinaForRanking model concatenates docs first, then query.
# Let's stay consistent with this novel design.
query_embeds = embeds[-1]
doc_embeds = embeds[:-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
for score in scores:
final_res_batch.append(
PoolingRequestOutput(
request_id=outputs[i].request_id,
outputs=score,
prompt_token_ids=outputs[i].prompt_token_ids,
num_cached_tokens=outputs[i].num_cached_tokens,
finished=True,
)
)
return final_res_batch
ScoringIOProcessors: dict[str, type[ScoringIOProcessor]] = {
p.name: p
for p in [
BiEncoderIOProcessor,
LateInteractionIOProcessor,
JinaRankingIOProcessor,
FlashLateInteractionIOProcessor,
CrossEncoderIOProcessor,
]
+12 -15
View File
@@ -4,7 +4,6 @@
from fastapi.responses import JSONResponse, Response
from vllm import PoolingParams
from vllm.config import VllmConfig
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.engine.protocol import UsageInfo
from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor
@@ -42,25 +41,23 @@ class ServingScores(PoolingServing):
enable_flash_late_interaction: bool = True,
**kwargs,
):
self.score_type = engine_client.model_config.score_type
self.io_processor_name: str = engine_client.model_config.score_type
self.enable_flash_late_interaction = (
self.score_type == "late-interaction" and enable_flash_late_interaction
self.io_processor_name == "late-interaction"
and enable_flash_late_interaction
)
if self.enable_flash_late_interaction:
self.io_processor_name = "flash-late-interaction"
if engine_client.model_config.architecture == "JinaForRanking":
self.io_processor_name = "jina-reranking-scoring"
self.enable_flash_late_interaction = False
super().__init__(engine_client, *args, **kwargs)
def init_io_processor(
self, vllm_config: VllmConfig, *args, **kwargs
) -> PoolingIOProcessor:
model_config = vllm_config.model_config
score_type: str = model_config.score_type
if self.enable_flash_late_interaction:
score_type = "flash-late-interaction"
assert score_type in ScoringIOProcessors
processor_cls = ScoringIOProcessors[score_type]
return processor_cls(vllm_config, *args, **kwargs)
def init_io_processor(self, *args, **kwargs) -> PoolingIOProcessor:
return ScoringIOProcessors[self.io_processor_name](*args, **kwargs)
async def __call__(self, *args, **kwargs) -> Response:
if not self.enable_flash_late_interaction: