[Bugfix][Pooling] Align CrossEncoder token type ids after truncation (#47772)
Signed-off-by: Ting Sun <suntcrick@gmail.com>
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import weakref
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -9,7 +10,10 @@ import torch
|
||||
from tests.models.utils import softmax
|
||||
from vllm import LLM, PoolingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import CrossEncoderIOProcessor
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoringData
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
|
||||
PROMPT = "The chef prepared a delicious meal."
|
||||
@@ -141,6 +145,45 @@ def test_max_tokens_per_doc(llm: LLM):
|
||||
assert with_limit_tokens < no_limit_tokens
|
||||
|
||||
|
||||
def test_token_type_ids_follow_post_tokenization():
|
||||
processor = object.__new__(CrossEncoderIOProcessor)
|
||||
processor.tokenizer = SimpleNamespace(truncation_side="right", pad_token_id=-1)
|
||||
processor.renderer = SimpleNamespace(process_for_engine=lambda prompt, _: prompt)
|
||||
processor.model_config = None
|
||||
processor.get_score_prompt = lambda **_: (
|
||||
"",
|
||||
{
|
||||
"prompt_token_ids": list(range(32)),
|
||||
"token_type_ids": [0] * 16 + [1] * 16,
|
||||
},
|
||||
)
|
||||
|
||||
engine_inputs, pooling_params = processor._pre_process(
|
||||
ScoringData(data_1=["query"], data_2=["document"]),
|
||||
TokenizeParams(
|
||||
max_total_tokens=None,
|
||||
truncate_prompt_tokens=16,
|
||||
truncation_side="left",
|
||||
),
|
||||
PoolingParams(task="classify", extra_kwargs={"cache_salt": "salt"}),
|
||||
)
|
||||
|
||||
assert engine_inputs[0]["prompt_token_ids"] == list(range(16, 32))
|
||||
assert pooling_params[0].extra_kwargs == {
|
||||
"cache_salt": "salt",
|
||||
"compressed_token_type_ids": 0,
|
||||
}
|
||||
|
||||
engine_inputs, pooling_params = processor._pre_process(
|
||||
ScoringData(data_1=["query"], data_2=["document"]),
|
||||
TokenizeParams(max_total_tokens=None, pad_prompt_tokens=40),
|
||||
PoolingParams(task="classify"),
|
||||
)
|
||||
|
||||
assert engine_inputs[0]["prompt_token_ids"] == list(range(32)) + [-1] * 8
|
||||
assert pooling_params[0].extra_kwargs == {"compressed_token_type_ids": 16}
|
||||
|
||||
|
||||
def test_pooling_params(llm: LLM):
|
||||
def get_outputs(use_activation):
|
||||
outputs = llm.score(
|
||||
|
||||
@@ -36,6 +36,39 @@ from .utils import (
|
||||
ScoringServeContext: TypeAlias = PoolingServeContext[ScoringRequest]
|
||||
|
||||
|
||||
def _apply_post_tokenization_to_token_type_ids(
|
||||
tokenizer: Any,
|
||||
tok_params: TokenizeParams,
|
||||
token_type_ids: list[int],
|
||||
) -> list[int]:
|
||||
pad_length = tok_params.pad_prompt_tokens
|
||||
if pad_length is not None and pad_length < 0:
|
||||
pad_length = tok_params.max_input_tokens
|
||||
|
||||
if pad_length is not None and pad_length > len(token_type_ids):
|
||||
pad_token_type_id = token_type_ids[-1] if token_type_ids else 0
|
||||
token_type_ids = token_type_ids + [pad_token_type_id] * (
|
||||
pad_length - len(token_type_ids)
|
||||
)
|
||||
|
||||
max_length = tok_params.truncate_prompt_tokens
|
||||
if max_length is not None and max_length < 0:
|
||||
max_length = tok_params.max_input_tokens
|
||||
|
||||
if max_length is None or max_length >= len(token_type_ids):
|
||||
return token_type_ids
|
||||
if max_length == 0:
|
||||
return token_type_ids[:0]
|
||||
|
||||
side = tok_params.truncation_side or (
|
||||
tokenizer.truncation_side if tokenizer is not None else None
|
||||
)
|
||||
if side == "left":
|
||||
return token_type_ids[-max_length:]
|
||||
|
||||
return token_type_ids[:max_length]
|
||||
|
||||
|
||||
class ScoringIOProcessor(PoolingIOProcessor):
|
||||
name: str
|
||||
pooling_task: PoolingTask
|
||||
@@ -469,9 +502,16 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
else None,
|
||||
)
|
||||
|
||||
if token_type_ids := engine_prompt.pop("token_type_ids", None):
|
||||
token_type_ids = engine_prompt.pop("token_type_ids", None)
|
||||
tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
|
||||
|
||||
if token_type_ids is not None:
|
||||
params = pooling_params.clone()
|
||||
compressed = compress_token_type_ids(token_type_ids)
|
||||
compressed = compress_token_type_ids(
|
||||
_apply_post_tokenization_to_token_type_ids(
|
||||
self.tokenizer, tok_params, token_type_ids
|
||||
)
|
||||
)
|
||||
params.extra_kwargs = {
|
||||
**(params.extra_kwargs or {}),
|
||||
"compressed_token_type_ids": compressed,
|
||||
@@ -480,7 +520,6 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
else:
|
||||
pooling_params_list.append(pooling_params)
|
||||
|
||||
tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
|
||||
if engine_prompt_extras:
|
||||
target_prompt = extract_target_prompt(self.model_config, engine_prompt)
|
||||
target_prompt.update(engine_prompt_extras)
|
||||
|
||||
Reference in New Issue
Block a user