[Bugfix] Preserve padding token types for cross encoders (#58143)
Signed-off-by: zhaoye <772971548@qq.com> Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -31,6 +31,7 @@ class _DummyTokenizer:
|
||||
# Outside the range of the prompt ids below, so a test can tell a pad
|
||||
# token apart from a real one.
|
||||
pad_token_id: int = 99999
|
||||
pad_token_type_id: int = 0
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -83,10 +84,11 @@ def test_llm_reranker_tokenization_is_independent_of_nonbinding_doc_limit(
|
||||
def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
|
||||
"""The cross-encoder segment boundary must survive truncate + pad.
|
||||
|
||||
`token_type_ids` are parallel to `prompt_token_ids` and are reduced to a
|
||||
single boundary index by `compress_token_type_ids`. If the two arrays are
|
||||
truncated and padded in different orders they no longer describe the same
|
||||
positions, and the model is told the query segment is empty.
|
||||
`token_type_ids` are parallel to `prompt_token_ids` and are reduced to the
|
||||
document segment's start and end by `compress_token_type_ids`. If the two
|
||||
arrays are truncated and padded in different orders they no longer
|
||||
describe the same positions, and the model is told the query segment is
|
||||
empty.
|
||||
"""
|
||||
tokenizer = _DummyTokenizer()
|
||||
num_query, num_doc = 20, 30
|
||||
@@ -114,10 +116,34 @@ def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
|
||||
|
||||
# Keeping the last 40 tokens drops the first 10 query tokens, so 10 query
|
||||
# tokens survive and the document starts at index 10.
|
||||
first_doc = compress_token_type_ids(token_type_ids)
|
||||
assert first_doc == 10
|
||||
assert prompt_token_ids[:first_doc] == list(range(10, num_query))
|
||||
assert prompt_token_ids[first_doc:40] == list(range(num_query, 50))
|
||||
doc_start, doc_end = compress_token_type_ids(token_type_ids)
|
||||
assert (doc_start, doc_end) == (10, 40)
|
||||
assert prompt_token_ids[:doc_start] == list(range(10, num_query))
|
||||
assert prompt_token_ids[doc_start:doc_end] == list(range(num_query, 50))
|
||||
assert token_type_ids[doc_end:] == [tokenizer.pad_token_type_id] * 60
|
||||
|
||||
|
||||
def test_token_type_padding_uses_the_tokenizer_pad_type():
|
||||
tokenizer = _DummyTokenizer()
|
||||
tok_params = TokenizeParams(
|
||||
max_total_tokens=8,
|
||||
pad_prompt_tokens=8,
|
||||
truncate_prompt_tokens=None,
|
||||
truncation_side="right",
|
||||
)
|
||||
|
||||
token_type_ids = _apply_post_tokenization_to_token_type_ids(
|
||||
tokenizer, tok_params, [0, 0, 0, 1, 1]
|
||||
)
|
||||
|
||||
assert token_type_ids == [0, 0, 0, 1, 1, 0, 0, 0]
|
||||
assert compress_token_type_ids(token_type_ids) == (3, 5)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("token_type_ids", [[0, 1, 0, 1], [0, 2]])
|
||||
def test_compress_token_type_ids_rejects_invalid_sequences(token_type_ids):
|
||||
with pytest.raises(ValueError, match="optional trailing zeros"):
|
||||
compress_token_type_ids(token_type_ids)
|
||||
|
||||
|
||||
def _pooling_output(
|
||||
|
||||
@@ -84,7 +84,7 @@ def _apply_post_tokenization_to_token_type_ids(
|
||||
pad_length = tok_params.max_input_tokens
|
||||
|
||||
if pad_length is not None and pad_length > len(token_type_ids):
|
||||
pad_token_type_id = token_type_ids[-1] if token_type_ids else 0
|
||||
pad_token_type_id = getattr(tokenizer, "pad_token_type_id", 0) or 0
|
||||
token_type_ids = token_type_ids + [pad_token_type_id] * (
|
||||
pad_length - len(token_type_ids)
|
||||
)
|
||||
|
||||
@@ -262,21 +262,30 @@ def parse_score_data(
|
||||
return prompt_1, prompt_2, mm_items, mm_uuids
|
||||
|
||||
|
||||
def compress_token_type_ids(token_type_ids: list[int]) -> int:
|
||||
"""Return position of the first 1 or the length of the list
|
||||
if not found.
|
||||
def compress_token_type_ids(token_type_ids: list[int]) -> tuple[int, int]:
|
||||
"""Return the half-open range containing token type 1.
|
||||
|
||||
Right padding uses the tokenizer's padding token type, which is commonly
|
||||
zero. Therefore, a padded query-document pair can contain zeros followed
|
||||
by ones followed by trailing zeros.
|
||||
"""
|
||||
first_one = len(token_type_ids)
|
||||
first_zero_after_one = len(token_type_ids)
|
||||
err_msg = (
|
||||
"Token type ids are expected to be a sequence"
|
||||
" of zeros followed by a sequence of ones"
|
||||
" of zeros followed by ones and optional trailing zeros"
|
||||
)
|
||||
for i, type_id in enumerate(token_type_ids):
|
||||
if type_id == 0 and first_one < i:
|
||||
raise ValueError(err_msg)
|
||||
elif type_id == 1 and first_one > i:
|
||||
first_one = i
|
||||
elif type_id > 1:
|
||||
if type_id not in (0, 1):
|
||||
raise ValueError(err_msg)
|
||||
if type_id == 1:
|
||||
if first_zero_after_one < len(token_type_ids):
|
||||
raise ValueError(err_msg)
|
||||
if first_one == len(token_type_ids):
|
||||
first_one = i
|
||||
elif first_one < len(token_type_ids) and first_zero_after_one == len(
|
||||
token_type_ids
|
||||
):
|
||||
first_zero_after_one = i
|
||||
|
||||
return first_one
|
||||
return first_one, first_zero_after_one
|
||||
|
||||
@@ -105,12 +105,15 @@ class EncoderOnlyModelState(DefaultModelState):
|
||||
if pooling_params is None or pooling_params.extra_kwargs is None:
|
||||
return
|
||||
|
||||
token_type_start = pooling_params.extra_kwargs.get("compressed_token_type_ids")
|
||||
if token_type_start is not None:
|
||||
token_type_range = pooling_params.extra_kwargs.get("compressed_token_type_ids")
|
||||
if token_type_range is not None:
|
||||
assert new_req_data.prompt_token_ids is not None
|
||||
token_type_start, token_type_end = token_type_range
|
||||
positions = torch.arange(
|
||||
len(new_req_data.prompt_token_ids), dtype=torch.int32
|
||||
)
|
||||
self.token_type_ids[new_req_data.req_id] = (
|
||||
torch.arange(len(new_req_data.prompt_token_ids), dtype=torch.int32)
|
||||
>= token_type_start
|
||||
(positions >= token_type_start) & (positions < token_type_end)
|
||||
).to(torch.int32)
|
||||
|
||||
def remove_request(self, req_id: str) -> None:
|
||||
|
||||
@@ -1079,8 +1079,9 @@ class GPUModelRunner(
|
||||
|
||||
for i in range(num_reqs):
|
||||
seq_len_i = seq_lens_cpu[i]
|
||||
pos = token_type_id_requests.get(i, seq_len_i)
|
||||
ids = (torch.arange(seq_len_i) >= pos).int()
|
||||
start, end = token_type_id_requests.get(i, (seq_len_i, seq_len_i))
|
||||
positions = torch.arange(seq_len_i)
|
||||
ids = ((positions >= start) & (positions < end)).int()
|
||||
token_type_ids.append(ids)
|
||||
|
||||
token_type_ids_cpu = torch.empty(
|
||||
|
||||
Reference in New Issue
Block a user