[Chore] Use tokenizer.encode and tokenizer.decode directly (#29851)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-12-02 12:30:40 +00:00
committed by GitHub
parent 951445a52d
commit 68ffbca7e4
9 changed files with 36 additions and 40 deletions
+4
View File
@@ -4,6 +4,8 @@
import warnings
from typing import Any
from typing_extensions import deprecated
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
@@ -73,6 +75,7 @@ def __getattr__(name: str):
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@deprecated("Will be removed in v0.13. Please use `tokenizer.decode()` instead.")
def decode_tokens(
tokenizer: TokenizerLike,
token_ids: list[int],
@@ -94,6 +97,7 @@ def decode_tokens(
return tokenizer.decode(token_ids, **kw_args)
@deprecated("Will be removed in v0.13. Please use `tokenizer.encode()` instead.")
def encode_tokens(
tokenizer: TokenizerLike,
text: str,