[Model][VLM] Support JinaVL Reranker (#20260)
Signed-off-by: shineran96 <shinewang96@gmail.com>
This commit is contained in:
@@ -17,8 +17,11 @@ from vllm.entrypoints.openai.protocol import (ErrorResponse, RerankDocument,
|
||||
ScoreResponseData, UsageInfo)
|
||||
from vllm.entrypoints.openai.serving_engine import OpenAIServing
|
||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||
from vllm.entrypoints.score_utils import (_cosine_similarity,
|
||||
_validate_score_input_lens)
|
||||
from vllm.entrypoints.score_utils import (ScoreContentPartParam,
|
||||
ScoreMultiModalParam,
|
||||
_cosine_similarity,
|
||||
_validate_score_input_lens,
|
||||
get_score_prompt)
|
||||
from vllm.entrypoints.utils import _validate_truncation_size
|
||||
from vllm.inputs.data import TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
@@ -137,11 +140,34 @@ class ServingScores(OpenAIServing):
|
||||
|
||||
return final_res_batch
|
||||
|
||||
def _preprocess_score(
|
||||
self,
|
||||
request: Union[RerankRequest, ScoreRequest],
|
||||
tokenizer: AnyTokenizer,
|
||||
tokenization_kwargs: dict[str, Any],
|
||||
data_1: Union[str, ScoreContentPartParam],
|
||||
data_2: Union[str, ScoreContentPartParam],
|
||||
) -> tuple[str, TokensPrompt]:
|
||||
|
||||
model_config = self.model_config
|
||||
|
||||
full_prompt, engine_prompt = get_score_prompt(
|
||||
model_config=model_config,
|
||||
data_1=data_1,
|
||||
data_2=data_2,
|
||||
tokenizer=tokenizer,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
if request.mm_processor_kwargs is not None:
|
||||
engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
|
||||
|
||||
return full_prompt, engine_prompt
|
||||
|
||||
async def _cross_encoding_score(
|
||||
self,
|
||||
tokenizer: AnyTokenizer,
|
||||
texts_1: list[str],
|
||||
texts_2: list[str],
|
||||
data_1: Union[list[str], list[ScoreContentPartParam]],
|
||||
data_2: Union[list[str], list[ScoreContentPartParam]],
|
||||
request: Union[RerankRequest, ScoreRequest],
|
||||
request_id=str,
|
||||
tokenization_kwargs: Optional[dict[str, Any]] = None,
|
||||
@@ -154,46 +180,67 @@ class ServingScores(OpenAIServing):
|
||||
request_prompts: list[str] = []
|
||||
engine_prompts: list[TokensPrompt] = []
|
||||
|
||||
if len(texts_1) == 1:
|
||||
texts_1 = texts_1 * len(texts_2)
|
||||
|
||||
input_pairs = [(t1, t2) for t1, t2 in zip(texts_1, texts_2)]
|
||||
if len(data_1) == 1:
|
||||
data_1 = data_1 * len(data_2)
|
||||
|
||||
if isinstance(tokenizer, MistralTokenizer):
|
||||
raise ValueError(
|
||||
"MistralTokenizer not supported for cross-encoding")
|
||||
|
||||
tokenize_async = make_async(tokenizer.__call__,
|
||||
executor=self._tokenizer_executor)
|
||||
|
||||
tokenization_kwargs = tokenization_kwargs or {}
|
||||
use_pad_token = self.model_config.use_pad_token
|
||||
|
||||
if use_pad_token:
|
||||
# cross_encoder models defaults to using pad_token.
|
||||
tokenized_prompts = await asyncio.gather(
|
||||
*(tokenize_async(text=t1, text_pair=t2, **tokenization_kwargs)
|
||||
for t1, t2 in input_pairs))
|
||||
input_pairs = [(t1, t2) for t1, t2 in zip(data_1, data_2)]
|
||||
|
||||
if self.model_config.is_multimodal_model:
|
||||
|
||||
preprocess_async = make_async(self._preprocess_score,
|
||||
executor=self._tokenizer_executor)
|
||||
|
||||
preprocessed_prompts = await asyncio.gather(
|
||||
*(preprocess_async(request=request,
|
||||
tokenizer=tokenizer,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
data_1=t1,
|
||||
data_2=t2) for t1, t2 in input_pairs))
|
||||
|
||||
for full_prompt, engine_prompt in preprocessed_prompts:
|
||||
request_prompts.append(full_prompt)
|
||||
engine_prompts.append(engine_prompt)
|
||||
|
||||
else:
|
||||
# `llm as reranker` models defaults to not using pad_token.
|
||||
tokenized_prompts = await asyncio.gather(
|
||||
*(tokenize_async(text=t1 + t2, **tokenization_kwargs)
|
||||
for t1, t2 in input_pairs))
|
||||
tokenize_async = make_async(tokenizer.__call__,
|
||||
executor=self._tokenizer_executor)
|
||||
use_pad_token = self.model_config.use_pad_token
|
||||
|
||||
for prompt_inputs, (t1, t2) in zip(tokenized_prompts, input_pairs):
|
||||
sep_token = tokenizer.sep_token if (tokenizer.sep_token
|
||||
and use_pad_token) else ''
|
||||
request_prompt = f"{t1}{sep_token}{t2}"
|
||||
if use_pad_token:
|
||||
# cross_encoder models defaults to using pad_token.
|
||||
tokenized_prompts = await asyncio.gather(*(
|
||||
tokenize_async(
|
||||
text=t1, # type: ignore[arg-type]
|
||||
text_pair=t2, # type: ignore[arg-type]
|
||||
**tokenization_kwargs) for t1, t2 in input_pairs))
|
||||
else:
|
||||
# `llm as reranker` models defaults to not using pad_token.
|
||||
tokenized_prompts = await asyncio.gather(*(
|
||||
tokenize_async(
|
||||
text=t1 + # type: ignore[operator]
|
||||
t2,
|
||||
**tokenization_kwargs) for t1, t2 in input_pairs))
|
||||
|
||||
input_ids = prompt_inputs["input_ids"]
|
||||
text_token_prompt = \
|
||||
self._validate_input(request, input_ids, request_prompt)
|
||||
engine_prompt = TokensPrompt(
|
||||
prompt_token_ids=text_token_prompt["prompt_token_ids"],
|
||||
token_type_ids=prompt_inputs.get("token_type_ids"))
|
||||
for prompt_inputs, (t1, t2) in zip(tokenized_prompts, input_pairs):
|
||||
sep_token = tokenizer.sep_token if (tokenizer.sep_token
|
||||
and use_pad_token) else ''
|
||||
request_prompt = f"{t1}{sep_token}{t2}"
|
||||
|
||||
request_prompts.append(request_prompt)
|
||||
engine_prompts.append(engine_prompt)
|
||||
input_ids = prompt_inputs["input_ids"]
|
||||
text_token_prompt = \
|
||||
self._validate_input(request, input_ids, request_prompt)
|
||||
engine_prompt = TokensPrompt(
|
||||
prompt_token_ids=text_token_prompt["prompt_token_ids"],
|
||||
token_type_ids=prompt_inputs.get("token_type_ids"))
|
||||
|
||||
request_prompts.append(request_prompt)
|
||||
engine_prompts.append(engine_prompt)
|
||||
|
||||
# Schedule the request and get the result generator.
|
||||
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||
@@ -233,8 +280,8 @@ class ServingScores(OpenAIServing):
|
||||
|
||||
async def _run_scoring(
|
||||
self,
|
||||
texts_1: Union[str, list[str]],
|
||||
texts_2: Union[str, list[str]],
|
||||
data_1: Union[list[str], str, ScoreMultiModalParam],
|
||||
data_2: Union[list[str], str, ScoreMultiModalParam],
|
||||
request: Union[ScoreRequest, RerankRequest],
|
||||
request_id: str,
|
||||
raw_request: Optional[Request] = None,
|
||||
@@ -259,18 +306,29 @@ class ServingScores(OpenAIServing):
|
||||
trace_headers = (None if raw_request is None else await
|
||||
self._get_trace_headers(raw_request.headers))
|
||||
|
||||
if isinstance(texts_1, str):
|
||||
texts_1 = [texts_1]
|
||||
if isinstance(texts_2, str):
|
||||
texts_2 = [texts_2]
|
||||
if not self.model_config.is_multimodal_model and (isinstance(
|
||||
data_1, dict) or isinstance(data_2, dict)):
|
||||
raise ValueError(
|
||||
f"MultiModalParam is not supported for {self.model_config.architecture}" # noqa: E501
|
||||
)
|
||||
|
||||
_validate_score_input_lens(texts_1, texts_2)
|
||||
if isinstance(data_1, str):
|
||||
data_1 = [data_1]
|
||||
elif isinstance(data_1, dict):
|
||||
data_1 = data_1.get("content") # type: ignore[assignment]
|
||||
|
||||
if isinstance(data_2, str):
|
||||
data_2 = [data_2]
|
||||
elif isinstance(data_2, dict):
|
||||
data_2 = data_2.get("content") # type: ignore[assignment]
|
||||
|
||||
_validate_score_input_lens(data_1, data_2) # type: ignore[arg-type]
|
||||
|
||||
if self.model_config.is_cross_encoder:
|
||||
return await self._cross_encoding_score(
|
||||
tokenizer=tokenizer,
|
||||
texts_1=texts_1,
|
||||
texts_2=texts_2,
|
||||
data_1=data_1, # type: ignore[arg-type]
|
||||
data_2=data_2, # type: ignore[arg-type]
|
||||
request=request,
|
||||
request_id=request_id,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
@@ -281,8 +339,8 @@ class ServingScores(OpenAIServing):
|
||||
else:
|
||||
return await self._embedding_score(
|
||||
tokenizer=tokenizer,
|
||||
texts_1=texts_1,
|
||||
texts_2=texts_2,
|
||||
texts_1=data_1, # type: ignore[arg-type]
|
||||
texts_2=data_2, # type: ignore[arg-type]
|
||||
request=request,
|
||||
request_id=request_id,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
@@ -349,7 +407,9 @@ class ServingScores(OpenAIServing):
|
||||
|
||||
request_id = f"rerank-{self._base_request_id(raw_request)}"
|
||||
documents = request.documents
|
||||
top_n = request.top_n if request.top_n > 0 else len(documents)
|
||||
top_n = request.top_n if request.top_n > 0 else (
|
||||
len(documents)
|
||||
if isinstance(documents, list) else len(documents["content"]))
|
||||
|
||||
try:
|
||||
final_res_batch = await self._run_scoring(
|
||||
@@ -410,7 +470,7 @@ class ServingScores(OpenAIServing):
|
||||
|
||||
def request_output_to_rerank_response(
|
||||
self, final_res_batch: list[PoolingRequestOutput], request_id: str,
|
||||
model_name: str, documents: list[str],
|
||||
model_name: str, documents: Union[list[str], ScoreMultiModalParam],
|
||||
top_n: int) -> RerankResponse:
|
||||
"""
|
||||
Convert the output of do_rank to a RerankResponse
|
||||
@@ -422,7 +482,9 @@ class ServingScores(OpenAIServing):
|
||||
|
||||
result = RerankResult(
|
||||
index=idx,
|
||||
document=RerankDocument(text=documents[idx]),
|
||||
document=RerankDocument(text=documents[idx]) if isinstance(
|
||||
documents, list) else RerankDocument(
|
||||
multi_modal=documents["content"][idx]),
|
||||
relevance_score=classify_res.outputs.score,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
Reference in New Issue
Block a user