[Refactor] TokenizerRegistry only uses lazy imports (#30609)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -97,7 +97,7 @@ class TokenizerLike(Protocol):
|
||||
messages: list["ChatCompletionMessageParam"],
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
**kwargs,
|
||||
) -> list[int]:
|
||||
) -> str | list[int]:
|
||||
raise NotImplementedError
|
||||
|
||||
def convert_tokens_to_string(self, tokens: list[str]) -> str:
|
||||
|
||||
Reference in New Issue
Block a user