[Bugfix][Frontend] Honor the constructor chat template in offline scoring (#61990)
Signed-off-by: mottopanikeiku <176798723+mottopanikeiku@users.noreply.github.com> Co-authored-by: mottopanikeiku <176798723+mottopanikeiku@users.noreply.github.com>
This commit is contained in:
@@ -10,7 +10,7 @@ import torch
|
||||
from tokenizers import Tokenizer, models, pre_tokenizers, processors
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm import TokensPrompt
|
||||
from vllm import PoolingParams, TokensPrompt
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import (
|
||||
BiEncoderIOProcessor,
|
||||
CrossEncoderIOProcessor,
|
||||
@@ -18,7 +18,9 @@ from vllm.entrypoints.pooling.scoring.io_processor import (
|
||||
LateInteractionIOProcessor,
|
||||
_apply_post_tokenization_to_token_type_ids,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoringData
|
||||
from vllm.entrypoints.pooling.scoring.utils import compress_token_type_ids
|
||||
from vllm.entrypoints.pooling.typing import OfflineScoringInputsContext
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput, RequestError
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
@@ -81,6 +83,49 @@ def test_llm_reranker_tokenization_is_independent_of_nonbinding_doc_limit(
|
||||
assert nonbinding["prompt_token_ids"] == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"default_template,request_template,expected",
|
||||
[
|
||||
("{{ messages[0].content }} {{ messages[1].content }}", None, "a bc"),
|
||||
(
|
||||
"{{ messages[0].content }} {{ messages[1].content }}",
|
||||
"{{ messages[1].content }} {{ messages[0].content }}",
|
||||
"bc a",
|
||||
),
|
||||
(None, "{{ messages[1].content }} {{ messages[0].content }}", "bc a"),
|
||||
(None, None, "abc"),
|
||||
],
|
||||
)
|
||||
def test_offline_scoring_uses_default_template_unless_overridden(
|
||||
llm_reranker_processor, default_template, request_template, expected
|
||||
):
|
||||
processor = llm_reranker_processor
|
||||
processor.chat_template = default_template
|
||||
processor.renderer = SimpleNamespace(
|
||||
default_cmpl_tok_params=TokenizeParams(max_total_tokens=32)
|
||||
)
|
||||
ctx = OfflineScoringInputsContext(
|
||||
pooling_task="classify",
|
||||
tokenization_kwargs=None,
|
||||
lora_request=None,
|
||||
priorities=None,
|
||||
scoring_data=ScoringData(data_1=["a"], data_2=["bc"]),
|
||||
chat_template=request_template,
|
||||
pooling_params=PoolingParams(),
|
||||
)
|
||||
factory, _ = processor.get_request_factory_offline(ctx)
|
||||
(request,) = factory()
|
||||
text, prompt = processor.get_score_prompt(
|
||||
request["data_1"],
|
||||
request["data_2"],
|
||||
request["tok_params"].get_encode_kwargs(),
|
||||
chat_template=request["chat_template"],
|
||||
)
|
||||
|
||||
assert text == expected
|
||||
assert prompt["prompt_token_ids"] == processor.tokenizer.encode(expected)
|
||||
|
||||
|
||||
def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
|
||||
"""The cross-encoder segment boundary must survive truncate + pad.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user