[Bugfix]: Fix TokenizerLike interface (#30009)
Signed-off-by: Rohan138 <rohanpotdar138@gmail.com>
This commit is contained in:
@@ -22,6 +22,9 @@ class TokenizerLike(Protocol):
|
||||
) -> "TokenizerLike":
|
||||
raise NotImplementedError
|
||||
|
||||
def num_special_tokens_to_add(self) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def all_special_tokens(self) -> list[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
Reference in New Issue
Block a user