[Bugfix][Frontend] Honor the constructor chat template in offline scoring (#61990)

Signed-off-by: mottopanikeiku <176798723+mottopanikeiku@users.noreply.github.com>
Co-authored-by: mottopanikeiku <176798723+mottopanikeiku@users.noreply.github.com>
This commit is contained in:
alp
2026-10-16 10:20:46 +00:00
committed by GitHub
co-authored by mottopanikeiku
parent 024c5188bf
commit 4ab99ff136
4 changed files with 55 additions and 4 deletions
@@ -10,7 +10,7 @@ import torch
from tokenizers import Tokenizer, models, pre_tokenizers, processors
from transformers import TokenizersBackend
from vllm import TokensPrompt
from vllm import PoolingParams, TokensPrompt
from vllm.entrypoints.pooling.scoring.io_processor import (
BiEncoderIOProcessor,
CrossEncoderIOProcessor,
@@ -18,7 +18,9 @@ from vllm.entrypoints.pooling.scoring.io_processor import (
LateInteractionIOProcessor,
_apply_post_tokenization_to_token_type_ids,
)
from vllm.entrypoints.pooling.scoring.typing import ScoringData
from vllm.entrypoints.pooling.scoring.utils import compress_token_type_ids
from vllm.entrypoints.pooling.typing import OfflineScoringInputsContext
from vllm.outputs import PoolingOutput, PoolingRequestOutput, RequestError
from vllm.renderers import TokenizeParams
@@ -81,6 +83,49 @@ def test_llm_reranker_tokenization_is_independent_of_nonbinding_doc_limit(
assert nonbinding["prompt_token_ids"] == expected
@pytest.mark.parametrize(
"default_template,request_template,expected",
[
("{{ messages[0].content }} {{ messages[1].content }}", None, "a bc"),
(
"{{ messages[0].content }} {{ messages[1].content }}",
"{{ messages[1].content }} {{ messages[0].content }}",
"bc a",
),
(None, "{{ messages[1].content }} {{ messages[0].content }}", "bc a"),
(None, None, "abc"),
],
)
def test_offline_scoring_uses_default_template_unless_overridden(
llm_reranker_processor, default_template, request_template, expected
):
processor = llm_reranker_processor
processor.chat_template = default_template
processor.renderer = SimpleNamespace(
default_cmpl_tok_params=TokenizeParams(max_total_tokens=32)
)
ctx = OfflineScoringInputsContext(
pooling_task="classify",
tokenization_kwargs=None,
lora_request=None,
priorities=None,
scoring_data=ScoringData(data_1=["a"], data_2=["bc"]),
chat_template=request_template,
pooling_params=PoolingParams(),
)
factory, _ = processor.get_request_factory_offline(ctx)
(request,) = factory()
text, prompt = processor.get_score_prompt(
request["data_1"],
request["data_2"],
request["tok_params"].get_encode_kwargs(),
chat_template=request["chat_template"],
)
assert text == expected
assert prompt["prompt_token_ids"] == processor.tokenizer.encode(expected)
def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
"""The cross-encoder segment boundary must survive truncate + pad.