Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -30,8 +30,9 @@ def _convert_tokens_to_string_with_added_encoders(
|
||||
current_sub_text: list[str] = []
|
||||
convert_tokens_to_string = tokenizer.convert_tokens_to_string
|
||||
added_vocab_set = set(tokenizer.get_added_vocab())
|
||||
all_special_tokens = set(
|
||||
tokenizer.all_special_tokens) if skip_special_tokens else ()
|
||||
all_special_tokens = (
|
||||
set(tokenizer.all_special_tokens) if skip_special_tokens else ()
|
||||
)
|
||||
|
||||
for token in output_tokens:
|
||||
# Use precomputed set for skip-special check
|
||||
@@ -70,11 +71,11 @@ def convert_prompt_ids_to_tokens(
|
||||
# We do not need to convert the whole prompt to tokens.
|
||||
# Offset a little more in case we have special tokens.
|
||||
new_tokens = tokenizer.convert_ids_to_tokens(
|
||||
prompt_ids[-INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET - 2:],
|
||||
skip_special_tokens=skip_special_tokens)
|
||||
prompt_ids[-INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET - 2 :],
|
||||
skip_special_tokens=skip_special_tokens,
|
||||
)
|
||||
read_offset = len(new_tokens)
|
||||
prefix_offset = max(
|
||||
read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
|
||||
prefix_offset = max(read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
|
||||
# This is required to guard against out-of-vocab prompt token ids
|
||||
_replace_none_with_empty(new_tokens) # type: ignore[arg-type]
|
||||
return new_tokens, prefix_offset, read_offset
|
||||
@@ -92,7 +93,7 @@ def convert_ids_list_to_tokens(
|
||||
|
||||
Returns:
|
||||
Python list of token string representations
|
||||
|
||||
|
||||
"""
|
||||
token_str_lst = []
|
||||
for token_id in token_ids:
|
||||
@@ -144,18 +145,17 @@ def detokenize_incrementally(
|
||||
# This is the first iteration for this sequence
|
||||
is_first_iter = prev_tokens is None
|
||||
if is_first_iter:
|
||||
(prev_tokens, prefix_offset,
|
||||
read_offset) = convert_prompt_ids_to_tokens(
|
||||
tokenizer,
|
||||
all_input_ids[:-1],
|
||||
skip_special_tokens=skip_special_tokens)
|
||||
(prev_tokens, prefix_offset, read_offset) = convert_prompt_ids_to_tokens(
|
||||
tokenizer, all_input_ids[:-1], skip_special_tokens=skip_special_tokens
|
||||
)
|
||||
assert prev_tokens is not None
|
||||
|
||||
# If the new token id is out of bounds, return an empty string.
|
||||
if 0 <= new_token_id < len(tokenizer):
|
||||
# Put new_token_id in a list so skip_special_tokens is respected
|
||||
new_tokens = tokenizer.convert_ids_to_tokens(
|
||||
[new_token_id], skip_special_tokens=skip_special_tokens)
|
||||
[new_token_id], skip_special_tokens=skip_special_tokens
|
||||
)
|
||||
if isinstance(new_tokens, str):
|
||||
new_tokens = [new_tokens]
|
||||
else:
|
||||
@@ -171,9 +171,9 @@ def detokenize_incrementally(
|
||||
# surrounding ids.
|
||||
if tokenizer.is_fast or not tokenizer.get_added_vocab():
|
||||
prefix_text = tokenizer.convert_tokens_to_string(
|
||||
output_tokens[prefix_offset:read_offset])
|
||||
new_text = tokenizer.convert_tokens_to_string(
|
||||
output_tokens[prefix_offset:])
|
||||
output_tokens[prefix_offset:read_offset]
|
||||
)
|
||||
new_text = tokenizer.convert_tokens_to_string(output_tokens[prefix_offset:])
|
||||
else:
|
||||
prefix_text = _convert_tokens_to_string_with_added_encoders(
|
||||
tokenizer,
|
||||
@@ -195,5 +195,5 @@ def detokenize_incrementally(
|
||||
# by the model
|
||||
return new_tokens, "", prefix_offset, read_offset
|
||||
|
||||
new_text = new_text[len(prefix_text):]
|
||||
new_text = new_text[len(prefix_text) :]
|
||||
return new_tokens, new_text, read_offset, len(output_tokens)
|
||||
|
||||
Reference in New Issue
Block a user