Convert formatting to use ruff instead of yapf + isort (#26247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 07:06:22 -07:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115205 additions and 94107 deletions
+17 -17
View File
@@ -30,8 +30,9 @@ def _convert_tokens_to_string_with_added_encoders(
current_sub_text: list[str] = []
convert_tokens_to_string = tokenizer.convert_tokens_to_string
added_vocab_set = set(tokenizer.get_added_vocab())
all_special_tokens = set(
tokenizer.all_special_tokens) if skip_special_tokens else ()
all_special_tokens = (
set(tokenizer.all_special_tokens) if skip_special_tokens else ()
)
for token in output_tokens:
# Use precomputed set for skip-special check
@@ -70,11 +71,11 @@ def convert_prompt_ids_to_tokens(
# We do not need to convert the whole prompt to tokens.
# Offset a little more in case we have special tokens.
new_tokens = tokenizer.convert_ids_to_tokens(
prompt_ids[-INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET - 2:],
skip_special_tokens=skip_special_tokens)
prompt_ids[-INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET - 2 :],
skip_special_tokens=skip_special_tokens,
)
read_offset = len(new_tokens)
prefix_offset = max(
read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
prefix_offset = max(read_offset - INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET, 0)
# This is required to guard against out-of-vocab prompt token ids
_replace_none_with_empty(new_tokens) # type: ignore[arg-type]
return new_tokens, prefix_offset, read_offset
@@ -92,7 +93,7 @@ def convert_ids_list_to_tokens(
Returns:
Python list of token string representations
"""
token_str_lst = []
for token_id in token_ids:
@@ -144,18 +145,17 @@ def detokenize_incrementally(
# This is the first iteration for this sequence
is_first_iter = prev_tokens is None
if is_first_iter:
(prev_tokens, prefix_offset,
read_offset) = convert_prompt_ids_to_tokens(
tokenizer,
all_input_ids[:-1],
skip_special_tokens=skip_special_tokens)
(prev_tokens, prefix_offset, read_offset) = convert_prompt_ids_to_tokens(
tokenizer, all_input_ids[:-1], skip_special_tokens=skip_special_tokens
)
assert prev_tokens is not None
# If the new token id is out of bounds, return an empty string.
if 0 <= new_token_id < len(tokenizer):
# Put new_token_id in a list so skip_special_tokens is respected
new_tokens = tokenizer.convert_ids_to_tokens(
[new_token_id], skip_special_tokens=skip_special_tokens)
[new_token_id], skip_special_tokens=skip_special_tokens
)
if isinstance(new_tokens, str):
new_tokens = [new_tokens]
else:
@@ -171,9 +171,9 @@ def detokenize_incrementally(
# surrounding ids.
if tokenizer.is_fast or not tokenizer.get_added_vocab():
prefix_text = tokenizer.convert_tokens_to_string(
output_tokens[prefix_offset:read_offset])
new_text = tokenizer.convert_tokens_to_string(
output_tokens[prefix_offset:])
output_tokens[prefix_offset:read_offset]
)
new_text = tokenizer.convert_tokens_to_string(output_tokens[prefix_offset:])
else:
prefix_text = _convert_tokens_to_string_with_added_encoders(
tokenizer,
@@ -195,5 +195,5 @@ def detokenize_incrementally(
# by the model
return new_tokens, "", prefix_offset, read_offset
new_text = new_text[len(prefix_text):]
new_text = new_text[len(prefix_text) :]
return new_tokens, new_text, read_offset, len(output_tokens)