diff --git a/docs/models/pooling_models/scoring.md b/docs/models/pooling_models/scoring.md index 0198810cbf..7ff186b5cd 100644 --- a/docs/models/pooling_models/scoring.md +++ b/docs/models/pooling_models/scoring.md @@ -455,6 +455,8 @@ Score templates are supported for **cross-encoder** models only. If you are usin Some scoring models require a specific prompt format to work correctly. You can specify a custom score template using the `--chat-template` parameter (see [Chat Template](../../serving/online_serving/README.md#chat-template)). +For offline scoring, pass `chat_template` to the `LLM` constructor to use it for every `LLM.score` call. A `chat_template` passed to `LLM.score` overrides the constructor's template for that call. + Like chat templates, the score template receives a `messages` list. For scoring, each message has a `role` attribute—either `"query"` or `"document"`. For the usual kind of point-wise cross-encoder, you can expect exactly two messages: one query and one document. To access the query and document content, use Jinja's `selectattr` filter: - **Query**: `{{ (messages | selectattr("role", "eq", "query") | first).content }}` diff --git a/tests/entrypoints/pooling/scoring/test_io_processor_unit.py b/tests/entrypoints/pooling/scoring/test_io_processor_unit.py index cd4d58fea6..7f2c157f25 100644 --- a/tests/entrypoints/pooling/scoring/test_io_processor_unit.py +++ b/tests/entrypoints/pooling/scoring/test_io_processor_unit.py @@ -10,7 +10,7 @@ import torch from tokenizers import Tokenizer, models, pre_tokenizers, processors from transformers import TokenizersBackend -from vllm import TokensPrompt +from vllm import PoolingParams, TokensPrompt from vllm.entrypoints.pooling.scoring.io_processor import ( BiEncoderIOProcessor, CrossEncoderIOProcessor, @@ -18,7 +18,9 @@ from vllm.entrypoints.pooling.scoring.io_processor import ( LateInteractionIOProcessor, _apply_post_tokenization_to_token_type_ids, ) +from vllm.entrypoints.pooling.scoring.typing import ScoringData from vllm.entrypoints.pooling.scoring.utils import compress_token_type_ids +from vllm.entrypoints.pooling.typing import OfflineScoringInputsContext from vllm.outputs import PoolingOutput, PoolingRequestOutput, RequestError from vllm.renderers import TokenizeParams @@ -81,6 +83,49 @@ def test_llm_reranker_tokenization_is_independent_of_nonbinding_doc_limit( assert nonbinding["prompt_token_ids"] == expected +@pytest.mark.parametrize( + "default_template,request_template,expected", + [ + ("{{ messages[0].content }} {{ messages[1].content }}", None, "a bc"), + ( + "{{ messages[0].content }} {{ messages[1].content }}", + "{{ messages[1].content }} {{ messages[0].content }}", + "bc a", + ), + (None, "{{ messages[1].content }} {{ messages[0].content }}", "bc a"), + (None, None, "abc"), + ], +) +def test_offline_scoring_uses_default_template_unless_overridden( + llm_reranker_processor, default_template, request_template, expected +): + processor = llm_reranker_processor + processor.chat_template = default_template + processor.renderer = SimpleNamespace( + default_cmpl_tok_params=TokenizeParams(max_total_tokens=32) + ) + ctx = OfflineScoringInputsContext( + pooling_task="classify", + tokenization_kwargs=None, + lora_request=None, + priorities=None, + scoring_data=ScoringData(data_1=["a"], data_2=["bc"]), + chat_template=request_template, + pooling_params=PoolingParams(), + ) + factory, _ = processor.get_request_factory_offline(ctx) + (request,) = factory() + text, prompt = processor.get_score_prompt( + request["data_1"], + request["data_2"], + request["tok_params"].get_encode_kwargs(), + chat_template=request["chat_template"], + ) + + assert text == expected + assert prompt["prompt_token_ids"] == processor.tokenizer.encode(expected) + + def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt(): """The cross-encoder segment boundary must survive truncate + pad. diff --git a/vllm/entrypoints/pooling/offline.py b/vllm/entrypoints/pooling/offline.py index edf92791a8..6764730869 100644 --- a/vllm/entrypoints/pooling/offline.py +++ b/vllm/entrypoints/pooling/offline.py @@ -325,8 +325,8 @@ class PoolingOfflineMixin(OfflineInferenceMixin): it is used to create the progress bar. If `False`, no progress bar is created. lora_request: LoRA request to use for generation, if any. - chat_template: The chat template to use for the scoring. If None, we - use the model's default chat template. + chat_template: The chat template to use for scoring. If None, use + the chat template passed to the LLM constructor. tokenization_kwargs: Overrides for `tokenizer.encode`. Returns: diff --git a/vllm/entrypoints/pooling/scoring/io_processor.py b/vllm/entrypoints/pooling/scoring/io_processor.py index f6678a7bbd..7c4a1fe917 100644 --- a/vllm/entrypoints/pooling/scoring/io_processor.py +++ b/vllm/entrypoints/pooling/scoring/io_processor.py @@ -548,7 +548,11 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): yield ScoringRenderParams( data_1=data_1[i], data_2=data_2[i], - chat_template=ctx.chat_template, + chat_template=( + self.chat_template + if ctx.chat_template is None + else ctx.chat_template + ), max_tokens_per_query=max_tokens_per_query, max_tokens_per_doc=max_tokens_per_doc, tok_params=tok_params,