[Pooling] Preserve reranker tokenization with document limits (#57666)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Wentao Ye
parent
50f78084da
commit
bb35e2587b
@@ -621,7 +621,7 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
model_config,
|
||||
)
|
||||
|
||||
# Apply truncation before defining closures
|
||||
# Limit the query and document separately before composing them.
|
||||
if max_tokens_per_query > 0 and isinstance(prompt_1, str):
|
||||
prompt_1 = truncate_text_to_tokens(
|
||||
prompt_1, tokenizer, max_tokens_per_query
|
||||
@@ -666,16 +666,8 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
full_prompt = tokenizer.decode(prompt_inputs["input_ids"])
|
||||
else:
|
||||
# `llm as reranker` defaults to not using separating token.
|
||||
if max_tokens_per_doc > 0 and isinstance(prompt_2, str):
|
||||
query_ids = tokenizer.encode(prompt_1, add_special_tokens=False)
|
||||
doc_ids = tokenizer.encode(prompt_2, add_special_tokens=False)
|
||||
doc_ids = doc_ids[:max_tokens_per_doc]
|
||||
input_ids = query_ids + doc_ids
|
||||
full_prompt = tokenizer.decode(input_ids)
|
||||
prompt_inputs = {"input_ids": input_ids}
|
||||
else:
|
||||
full_prompt = prompt_1 + prompt_2
|
||||
prompt_inputs = tokenizer(text=full_prompt, **local_kwargs)
|
||||
full_prompt = prompt_1 + prompt_2
|
||||
prompt_inputs = tokenizer(text=full_prompt, **local_kwargs)
|
||||
return full_prompt, prompt_inputs
|
||||
|
||||
# FIXME: For now, we only apply a template when one is explicitly provided.
|
||||
|
||||
@@ -48,13 +48,27 @@ def truncate_text_to_tokens(
|
||||
|
||||
Uses offset_mapping to slice the original text at the exact character
|
||||
boundary, avoiding lossy encode→decode round-trips that can shift
|
||||
the token count by 1-3 tokens due to BPE merge boundary changes.
|
||||
the token count due to BPE merge boundary changes.
|
||||
"""
|
||||
encoding = tokenizer(text, add_special_tokens=False, return_offsets_mapping=True)
|
||||
if len(encoding["input_ids"]) <= max_tokens:
|
||||
return text
|
||||
char_end = encoding["offset_mapping"][max_tokens - 1][1]
|
||||
return text[:char_end]
|
||||
if max_tokens <= 0:
|
||||
return ""
|
||||
|
||||
while text:
|
||||
encoding = tokenizer(
|
||||
text,
|
||||
add_special_tokens=False,
|
||||
return_offsets_mapping=True,
|
||||
)
|
||||
offsets = encoding["offset_mapping"]
|
||||
if len(encoding["input_ids"]) <= max_tokens:
|
||||
break
|
||||
|
||||
char_end = min(offsets[max_tokens - 1][1], offsets[max_tokens][0])
|
||||
if char_end >= len(text):
|
||||
char_end = len(text) - 1
|
||||
text = text[:char_end]
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def compute_maxsim_score(q_emb: torch.Tensor, d_emb: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
Reference in New Issue
Block a user