From 0303f37a54dbf0c81e375e654d431a9d33a855a9 Mon Sep 17 00:00:00 2001 From: Ting SUN Date: Wed, 8 Jul 2026 10:59:22 +0700 Subject: [PATCH] [Bugfix][Pooling] Align CrossEncoder token type ids after truncation (#47772) Signed-off-by: Ting Sun --- .../scoring/test_cross_encoder_offline.py | 43 ++++++++++++++++++ .../pooling/scoring/io_processor.py | 45 +++++++++++++++++-- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/tests/entrypoints/pooling/scoring/test_cross_encoder_offline.py b/tests/entrypoints/pooling/scoring/test_cross_encoder_offline.py index 56e83de3f7..df79a387af 100644 --- a/tests/entrypoints/pooling/scoring/test_cross_encoder_offline.py +++ b/tests/entrypoints/pooling/scoring/test_cross_encoder_offline.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import weakref +from types import SimpleNamespace import pytest import torch @@ -9,7 +10,10 @@ import torch from tests.models.utils import softmax from vllm import LLM, PoolingParams from vllm.distributed import cleanup_dist_env_and_memory +from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor +from vllm.entrypoints.pooling.scoring.typing import ScoringData from vllm.platforms import current_platform +from vllm.renderers import TokenizeParams MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls" PROMPT = "The chef prepared a delicious meal." @@ -141,6 +145,45 @@ def test_max_tokens_per_doc(llm: LLM): assert with_limit_tokens < no_limit_tokens +def test_token_type_ids_follow_post_tokenization(): + processor = object.__new__(CrossEncoderIOProcessor) + processor.tokenizer = SimpleNamespace(truncation_side="right", pad_token_id=-1) + processor.renderer = SimpleNamespace(process_for_engine=lambda prompt, _: prompt) + processor.model_config = None + processor.get_score_prompt = lambda **_: ( + "", + { + "prompt_token_ids": list(range(32)), + "token_type_ids": [0] * 16 + [1] * 16, + }, + ) + + engine_inputs, pooling_params = processor._pre_process( + ScoringData(data_1=["query"], data_2=["document"]), + TokenizeParams( + max_total_tokens=None, + truncate_prompt_tokens=16, + truncation_side="left", + ), + PoolingParams(task="classify", extra_kwargs={"cache_salt": "salt"}), + ) + + assert engine_inputs[0]["prompt_token_ids"] == list(range(16, 32)) + assert pooling_params[0].extra_kwargs == { + "cache_salt": "salt", + "compressed_token_type_ids": 0, + } + + engine_inputs, pooling_params = processor._pre_process( + ScoringData(data_1=["query"], data_2=["document"]), + TokenizeParams(max_total_tokens=None, pad_prompt_tokens=40), + PoolingParams(task="classify"), + ) + + assert engine_inputs[0]["prompt_token_ids"] == list(range(32)) + [-1] * 8 + assert pooling_params[0].extra_kwargs == {"compressed_token_type_ids": 16} + + def test_pooling_params(llm: LLM): def get_outputs(use_activation): outputs = llm.score( diff --git a/vllm/entrypoints/pooling/scoring/io_processor.py b/vllm/entrypoints/pooling/scoring/io_processor.py index bd5d8255d5..db872b7f30 100644 --- a/vllm/entrypoints/pooling/scoring/io_processor.py +++ b/vllm/entrypoints/pooling/scoring/io_processor.py @@ -36,6 +36,39 @@ from .utils import ( ScoringServeContext: TypeAlias = PoolingServeContext[ScoringRequest] +def _apply_post_tokenization_to_token_type_ids( + tokenizer: Any, + tok_params: TokenizeParams, + token_type_ids: list[int], +) -> list[int]: + pad_length = tok_params.pad_prompt_tokens + if pad_length is not None and pad_length < 0: + pad_length = tok_params.max_input_tokens + + if pad_length is not None and pad_length > len(token_type_ids): + pad_token_type_id = token_type_ids[-1] if token_type_ids else 0 + token_type_ids = token_type_ids + [pad_token_type_id] * ( + pad_length - len(token_type_ids) + ) + + max_length = tok_params.truncate_prompt_tokens + if max_length is not None and max_length < 0: + max_length = tok_params.max_input_tokens + + if max_length is None or max_length >= len(token_type_ids): + return token_type_ids + if max_length == 0: + return token_type_ids[:0] + + side = tok_params.truncation_side or ( + tokenizer.truncation_side if tokenizer is not None else None + ) + if side == "left": + return token_type_ids[-max_length:] + + return token_type_ids[:max_length] + + class ScoringIOProcessor(PoolingIOProcessor): name: str pooling_task: PoolingTask @@ -469,9 +502,16 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): else None, ) - if token_type_ids := engine_prompt.pop("token_type_ids", None): + token_type_ids = engine_prompt.pop("token_type_ids", None) + tok_params.apply_post_tokenization(self.tokenizer, engine_prompt) + + if token_type_ids is not None: params = pooling_params.clone() - compressed = compress_token_type_ids(token_type_ids) + compressed = compress_token_type_ids( + _apply_post_tokenization_to_token_type_ids( + self.tokenizer, tok_params, token_type_ids + ) + ) params.extra_kwargs = { **(params.extra_kwargs or {}), "compressed_token_type_ids": compressed, @@ -480,7 +520,6 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): else: pooling_params_list.append(pooling_params) - tok_params.apply_post_tokenization(self.tokenizer, engine_prompt) if engine_prompt_extras: target_prompt = extract_target_prompt(self.model_config, engine_prompt) target_prompt.update(engine_prompt_extras)