[Pooling] Preserve reranker tokenization with document limits (#57666)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-09-25 11:04:41 -04:00
committed by GitHub
co-authored by Wentao Ye
parent 50f78084da
commit bb35e2587b
4 changed files with 94 additions and 18 deletions
@@ -3,11 +3,15 @@
"""Unit tests for ScoringIOProcessor post-tokenization helpers."""
from dataclasses import dataclass
from types import SimpleNamespace
import pytest
from tokenizers import Tokenizer, models, pre_tokenizers, processors
from transformers import TokenizersBackend
from vllm import TokensPrompt
from vllm.entrypoints.pooling.scoring.io_processor import (
CrossEncoderIOProcessor,
_apply_post_tokenization_to_token_type_ids,
)
from vllm.entrypoints.pooling.scoring.utils import compress_token_type_ids
@@ -24,6 +28,53 @@ class _DummyTokenizer:
pad_token_id: int = 99999
@pytest.fixture
def llm_reranker_processor() -> CrossEncoderIOProcessor:
vocab = {
"[UNK]": 0,
"[CLS]": 1,
"[SEP]": 2,
"a": 3,
"bc": 4,
"##b": 5,
"##c": 6,
}
backend = Tokenizer(models.WordPiece(vocab, unk_token="[UNK]"))
backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
backend.post_processor = processors.TemplateProcessing(
single="[CLS] $A [SEP]",
special_tokens=[("[CLS]", 1), ("[SEP]", 2)],
)
processor = CrossEncoderIOProcessor.__new__(CrossEncoderIOProcessor)
processor.model_config = SimpleNamespace(enable_prompt_embeds=False)
processor.tokenizer = TokenizersBackend(
tokenizer_object=backend,
unk_token="[UNK]",
cls_token="[CLS]",
sep_token="[SEP]",
)
processor.supports_score_template = False
processor.use_sep_token = False
processor.model = None
return processor
def test_llm_reranker_tokenization_is_independent_of_nonbinding_doc_limit(
llm_reranker_processor: CrossEncoderIOProcessor,
):
encode_kwargs = {"add_special_tokens": True}
_, uncapped = llm_reranker_processor.get_score_prompt("a", "bc", encode_kwargs)
_, nonbinding = llm_reranker_processor.get_score_prompt(
"a", "bc", encode_kwargs, max_tokens_per_doc=10
)
expected = [1, 3, 5, 6, 2]
assert uncapped["prompt_token_ids"] == expected
assert nonbinding["prompt_token_ids"] == expected
def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
"""The cross-encoder segment boundary must survive truncate + pad.
@@ -2,9 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tokenizers import Tokenizer, models, pre_tokenizers
from transformers import TokenizersBackend
from vllm.entrypoints.pooling.scoring.typing import ScoreInput
from vllm.entrypoints.pooling.scoring.utils import validate_score_input
from vllm.entrypoints.pooling.scoring.utils import (
truncate_text_to_tokens,
validate_score_input,
)
from vllm.exceptions import VLLMValidationError
@@ -63,3 +68,17 @@ def test_validate_score_input_rejects_invalid_inputs(
assert str(exc_info.value) == message
assert exc_info.value.parameter is None
assert exc_info.value.value is None
def test_truncate_text_to_tokens_handles_shared_character_offsets():
vocab = {
token: i for i, token in enumerate(sorted(pre_tokenizers.ByteLevel.alphabet()))
}
backend = Tokenizer(models.BPE(vocab=vocab, merges=[]))
backend.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
tokenizer = TokenizersBackend(tokenizer_object=backend)
truncated = truncate_text_to_tokens("aéx", tokenizer, max_tokens=2)
assert truncated == "a"
assert len(tokenizer(truncated, add_special_tokens=False)["input_ids"]) == 1