[Chore] Use tokenizer.encode and tokenizer.decode directly (#29851)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -25,7 +25,6 @@ from typing_extensions import TypeVar, assert_never
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.transformers_utils.tokenizer import decode_tokens, encode_tokens
|
||||
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
|
||||
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
|
||||
from vllm.utils.jsontree import JSONTree, json_map_leaves
|
||||
@@ -80,9 +79,9 @@ def _cached_encode(
|
||||
tokenizer: TokenizerLike,
|
||||
text: str,
|
||||
*,
|
||||
add_special_tokens: bool | None = None,
|
||||
add_special_tokens: bool = True,
|
||||
) -> list[int]:
|
||||
return encode_tokens(tokenizer, text, add_special_tokens=add_special_tokens)
|
||||
return tokenizer.encode(text, add_special_tokens=add_special_tokens)
|
||||
|
||||
|
||||
@lru_cache(maxsize=2048)
|
||||
@@ -90,11 +89,9 @@ def _cached_decode(
|
||||
tokenizer: TokenizerLike,
|
||||
token_ids: tuple[int, ...],
|
||||
*,
|
||||
skip_special_tokens: bool | None = None,
|
||||
skip_special_tokens: bool = False,
|
||||
) -> str:
|
||||
return decode_tokens(
|
||||
tokenizer, list(token_ids), skip_special_tokens=skip_special_tokens
|
||||
)
|
||||
return tokenizer.decode(list(token_ids), skip_special_tokens=skip_special_tokens)
|
||||
|
||||
|
||||
def _seq2text(
|
||||
@@ -110,7 +107,7 @@ def _seq2text(
|
||||
raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")
|
||||
|
||||
if not use_cache:
|
||||
return decode_tokens(tokenizer, seq)
|
||||
return tokenizer.decode(seq)
|
||||
|
||||
return _cached_decode(tokenizer, tuple(seq))
|
||||
|
||||
@@ -126,7 +123,7 @@ def _seq2tokens(
|
||||
raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")
|
||||
|
||||
if not use_cache:
|
||||
return encode_tokens(tokenizer, seq, add_special_tokens=False)
|
||||
return tokenizer.encode(seq, add_special_tokens=False)
|
||||
|
||||
return _cached_encode(tokenizer, seq, add_special_tokens=False)
|
||||
|
||||
@@ -2198,8 +2195,8 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_data)
|
||||
if isinstance(decoder_prompt_raw, str):
|
||||
decoder_prompt_ids = encode_tokens(
|
||||
tokenizer, decoder_prompt_raw, add_special_tokens=False
|
||||
decoder_prompt_ids = tokenizer.encode(
|
||||
decoder_prompt_raw, add_special_tokens=False
|
||||
)
|
||||
else:
|
||||
decoder_prompt_ids = decoder_prompt_raw
|
||||
|
||||
Reference in New Issue
Block a user