[Frontend] Score with the chat template declared by Sentence Transformers CrossEncoders (#60877)

Signed-off-by: rdlhio <contact@rdlh.io>
This commit is contained in:
Rémi Delhaye
2026-10-18 11:30:12 +00:00
committed by GitHub
parent 953fa442fa
commit 3c13653738
7 changed files with 192 additions and 27 deletions
@@ -65,6 +65,8 @@ def llm_reranker_processor() -> CrossEncoderIOProcessor:
processor.supports_score_template = False
processor.use_sep_token = False
processor.model = None
processor.saved_chat_template = None
processor.saved_chat_template_kwargs = {}
return processor
@@ -39,6 +39,14 @@ RERANK_MODELS = [
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
),
# Declares its score template in its Sentence Transformers config, so no
# chat template is passed.
RerankModelInfo(
"nvidia/llama-nemotron-rerank-vl-1b-v2",
architecture="LlamaNemotronVLForSequenceClassification",
mteb_score=0.33919,
mteb_tol=1e-2,
),
]
@@ -10,7 +10,6 @@ Both variants share a SigLIP vision encoder with a bidirectional LLaMA backbone.
from collections.abc import Sequence
from io import BytesIO
from pathlib import Path
import pybase64 as base64
import pytest
@@ -202,13 +201,6 @@ def test_models(
RERANKER_MODELS = ["nvidia/llama-nemotron-rerank-vl-1b-v2"]
# The tokenizer's built-in chat template is not suitable for the Score/Rerank
# APIs (it's inherited from the base LLM). We must use the provided override.
_RERANKER_SCORE_TEMPLATE = (
Path(__file__).parents[4]
/ "examples/pooling/score/template/nemotron-vl-rerank.jinja"
).read_text()
RERANKER_TEXT_QUERY = "How is AI improving the intelligence and capabilities of robots?"
RERANKER_TEXT_DOCS = [
"AI enables robots to perceive, plan, and act autonomously.",
@@ -281,7 +273,11 @@ def _run_vllm_reranker(
dtype: str,
input_cases: Sequence[RerankerCase],
) -> list[list[float]]:
"""Run all vLLM reranker cases in one model lifecycle."""
"""Run all vLLM reranker cases in one model lifecycle.
The checkpoint declares its score template in its Sentence Transformers
config, so no chat template is passed.
"""
with vllm_runner(
model,
runner="pooling",
@@ -298,11 +294,7 @@ def _run_vllm_reranker(
if not has_images:
queries = [query] * len(docs)
doc_texts = [doc_text for doc_text, _ in docs]
outputs = vllm_model.score(
queries,
doc_texts,
chat_template=_RERANKER_SCORE_TEMPLATE,
)
outputs = vllm_model.score(queries, doc_texts)
else:
query_params = [
ScoreMultiModalParam(
@@ -334,11 +326,7 @@ def _run_vllm_reranker(
)
doc_params.append(ScoreMultiModalParam(content=content))
raw_outputs = vllm_model.llm.score(
query_params,
doc_params,
chat_template=_RERANKER_SCORE_TEMPLATE,
)
raw_outputs = vllm_model.llm.score(query_params, doc_params)
outputs = [output.outputs.score for output in raw_outputs]
scores_per_case.append(outputs)