[New Model]: jinaai/jina-reranker-v3 (#38800)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -24,7 +24,13 @@ from vllm.entrypoints.pooling.embed.protocol import (
|
||||
EmbeddingChatRequest,
|
||||
EmbeddingCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import JinaRankingIOProcessorMixin
|
||||
from vllm.entrypoints.pooling.typing import (
|
||||
OfflineInputsContext,
|
||||
PoolingChatLikeRequest,
|
||||
PoolingCompletionLikeRequest,
|
||||
PoolingServeContext,
|
||||
)
|
||||
from vllm.inputs import EngineInput, tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||
@@ -553,3 +559,48 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
|
||||
class TokenEmbedIOProcessor(PoolingIOProcessor):
|
||||
name = "token_embed"
|
||||
|
||||
|
||||
class JinaRankingTokenEmbedIOProcessor(
|
||||
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
|
||||
):
|
||||
def pre_process_online(self, ctx: PoolingServeContext):
|
||||
request = ctx.request
|
||||
if isinstance(request, PoolingCompletionLikeRequest):
|
||||
prompts = request.input
|
||||
if not isinstance(prompts, Sequence) or len(prompts) < 2:
|
||||
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
|
||||
|
||||
text_prompts = self.ensure_str(prompts)
|
||||
|
||||
# The JinaForRanking model concatenates docs first, then query.
|
||||
# Let's stay consistent with this novel design.
|
||||
prompt_input = self.format_docs_prompts_func(
|
||||
query=text_prompts[-1], docs=text_prompts[:-1]
|
||||
)
|
||||
|
||||
engine_inputs = self._preprocess_completion_online(
|
||||
request,
|
||||
prompt_input=prompt_input,
|
||||
prompt_embeds=None,
|
||||
)
|
||||
elif isinstance(request, PoolingChatLikeRequest):
|
||||
raise ValueError("The JinaForRanking does not support chat Request.")
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.name} request type")
|
||||
|
||||
ctx.engine_inputs = engine_inputs
|
||||
|
||||
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
|
||||
if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2:
|
||||
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
|
||||
|
||||
text_prompts = self.ensure_str(ctx.prompts)
|
||||
|
||||
# The JinaForRanking model concatenates docs first, then query.
|
||||
# Let's stay consistent with this novel design.
|
||||
ctx.prompts = self.format_docs_prompts_func(
|
||||
query=text_prompts[-1], docs=text_prompts[:-1]
|
||||
)
|
||||
|
||||
return super().pre_process_offline(ctx)
|
||||
|
||||
@@ -59,6 +59,13 @@ def init_pooling_io_processors(
|
||||
if score_type is not None and score_type in ScoringIOProcessors:
|
||||
processors[score_type] = ScoringIOProcessors[score_type]
|
||||
|
||||
if model_config.architecture == "JinaForRanking":
|
||||
from .embed.io_processor import JinaRankingTokenEmbedIOProcessor
|
||||
from .scoring.io_processor import ScoringIOProcessors
|
||||
|
||||
processors["token_embed"] = JinaRankingTokenEmbedIOProcessor
|
||||
processors["late-interaction"] = ScoringIOProcessors["jina-reranking-scoring"]
|
||||
|
||||
return {
|
||||
task: processor_cls(
|
||||
vllm_config=vllm_config,
|
||||
|
||||
@@ -416,11 +416,137 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
return full_prompt, engine_prompt
|
||||
|
||||
|
||||
class JinaRankingIOProcessorMixin:
|
||||
@staticmethod
|
||||
def sanitize_input(text: str, special_tokens: dict[str, str]) -> str:
|
||||
for token in special_tokens.values():
|
||||
text = text.replace(token, "")
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def format_docs_prompts_func(
|
||||
query: str,
|
||||
docs: list[str],
|
||||
special_tokens: dict[str, str] | None = None,
|
||||
instruction: str | None = None,
|
||||
no_thinking: bool = True,
|
||||
) -> str:
|
||||
# TODO: Try converting the code below into a chat template.
|
||||
|
||||
default_special_tokens = {
|
||||
"query_embed_token": "<|rerank_token|>",
|
||||
"doc_embed_token": "<|embed_token|>",
|
||||
}
|
||||
if special_tokens is None:
|
||||
special_tokens = default_special_tokens
|
||||
|
||||
query = JinaRankingIOProcessorMixin.sanitize_input(query, special_tokens)
|
||||
docs = [
|
||||
JinaRankingIOProcessorMixin.sanitize_input(doc, special_tokens)
|
||||
for doc in docs
|
||||
]
|
||||
|
||||
prefix = (
|
||||
"<|im_start|>system\n"
|
||||
"You are a search relevance expert who can determine a ranking of the passages based on how relevant they are to the query. " # noqa: E501
|
||||
"If the query is a question, how relevant a passage is depends on how well it answers the question. " # noqa: E501
|
||||
"If not, try to analyze the intent of the query and assess how well each passage satisfies the intent. " # noqa: E501
|
||||
"If an instruction is provided, you should follow the instruction when determining the ranking." # noqa: E501
|
||||
"<|im_end|>\n<|im_start|>user\n"
|
||||
)
|
||||
suffix = "<|im_end|>\n<|im_start|>assistant\n"
|
||||
if no_thinking:
|
||||
suffix += "<think>\n\n</think>\n\n"
|
||||
|
||||
doc_emb_token = special_tokens["doc_embed_token"]
|
||||
query_emb_token = special_tokens["query_embed_token"]
|
||||
|
||||
prompt = (
|
||||
f"I will provide you with {len(docs)} passages, each indicated by a numerical identifier. " # noqa: E501
|
||||
f"Rank the passages based on their relevance to query: {query}\n"
|
||||
)
|
||||
|
||||
if instruction:
|
||||
prompt += f"<instruct>\n{instruction}\n</instruct>\n"
|
||||
|
||||
doc_prompts = [
|
||||
f'<passage id="{i}">\n{doc}{doc_emb_token}\n</passage>'
|
||||
for i, doc in enumerate(docs)
|
||||
]
|
||||
prompt += "\n".join(doc_prompts) + "\n"
|
||||
prompt += f"<query>\n{query}{query_emb_token}\n</query>"
|
||||
|
||||
return prefix + prompt + suffix
|
||||
|
||||
@staticmethod
|
||||
def ensure_str(data: Sequence[Any]) -> list[str]:
|
||||
text: list[str] = []
|
||||
for prompt in data:
|
||||
if not isinstance(prompt, str):
|
||||
raise ValueError(
|
||||
"The JinaForRanking model only supports text as input."
|
||||
)
|
||||
text.append(prompt)
|
||||
return text
|
||||
|
||||
|
||||
class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorMixin):
|
||||
name = "jina-reranking-scoring"
|
||||
pooling_task: PoolingTask = "token_embed"
|
||||
|
||||
def _pre_process(
|
||||
self,
|
||||
scoring_data: ScoringData,
|
||||
tok_params: TokenizeParams,
|
||||
prompt_extras: dict[str, Any] | None = None,
|
||||
) -> Sequence[EngineInput]:
|
||||
queries = self.ensure_str(scoring_data.data_1)
|
||||
docs = self.ensure_str(scoring_data.data_2)
|
||||
|
||||
if len(queries) == 1:
|
||||
prompts = [self.format_docs_prompts_func(query=queries[0], docs=docs)]
|
||||
else:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(query=q, docs=[d])
|
||||
for q, d in zip(queries, docs)
|
||||
]
|
||||
|
||||
return self._preprocess_completion_offline(
|
||||
prompts=prompts, tok_params=tok_params, prompt_extras=prompt_extras
|
||||
)
|
||||
|
||||
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
|
||||
for i in range(len(outputs)):
|
||||
embeds = outputs[i].outputs.data.float()
|
||||
|
||||
# The JinaForRanking model concatenates docs first, then query.
|
||||
# Let's stay consistent with this novel design.
|
||||
query_embeds = embeds[-1]
|
||||
doc_embeds = embeds[:-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
|
||||
for score in scores:
|
||||
final_res_batch.append(
|
||||
PoolingRequestOutput(
|
||||
request_id=outputs[i].request_id,
|
||||
outputs=score,
|
||||
prompt_token_ids=outputs[i].prompt_token_ids,
|
||||
num_cached_tokens=outputs[i].num_cached_tokens,
|
||||
finished=True,
|
||||
)
|
||||
)
|
||||
return final_res_batch
|
||||
|
||||
|
||||
ScoringIOProcessors: dict[str, type[ScoringIOProcessor]] = {
|
||||
p.name: p
|
||||
for p in [
|
||||
BiEncoderIOProcessor,
|
||||
LateInteractionIOProcessor,
|
||||
JinaRankingIOProcessor,
|
||||
FlashLateInteractionIOProcessor,
|
||||
CrossEncoderIOProcessor,
|
||||
]
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import UsageInfo
|
||||
from vllm.entrypoints.pooling.base.io_processor import PoolingIOProcessor
|
||||
@@ -42,25 +41,23 @@ class ServingScores(PoolingServing):
|
||||
enable_flash_late_interaction: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
self.score_type = engine_client.model_config.score_type
|
||||
self.io_processor_name: str = engine_client.model_config.score_type
|
||||
self.enable_flash_late_interaction = (
|
||||
self.score_type == "late-interaction" and enable_flash_late_interaction
|
||||
self.io_processor_name == "late-interaction"
|
||||
and enable_flash_late_interaction
|
||||
)
|
||||
|
||||
if self.enable_flash_late_interaction:
|
||||
self.io_processor_name = "flash-late-interaction"
|
||||
|
||||
if engine_client.model_config.architecture == "JinaForRanking":
|
||||
self.io_processor_name = "jina-reranking-scoring"
|
||||
self.enable_flash_late_interaction = False
|
||||
|
||||
super().__init__(engine_client, *args, **kwargs)
|
||||
|
||||
def init_io_processor(
|
||||
self, vllm_config: VllmConfig, *args, **kwargs
|
||||
) -> PoolingIOProcessor:
|
||||
model_config = vllm_config.model_config
|
||||
|
||||
score_type: str = model_config.score_type
|
||||
if self.enable_flash_late_interaction:
|
||||
score_type = "flash-late-interaction"
|
||||
|
||||
assert score_type in ScoringIOProcessors
|
||||
processor_cls = ScoringIOProcessors[score_type]
|
||||
return processor_cls(vllm_config, *args, **kwargs)
|
||||
def init_io_processor(self, *args, **kwargs) -> PoolingIOProcessor:
|
||||
return ScoringIOProcessors[self.io_processor_name](*args, **kwargs)
|
||||
|
||||
async def __call__(self, *args, **kwargs) -> Response:
|
||||
if not self.enable_flash_late_interaction:
|
||||
|
||||
Reference in New Issue
Block a user