[Refactor] Remove dead code multiple places (#60120)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-10-10 15:41:10 +00:00
committed by GitHub
parent 838526c7e5
commit 3709632ff2
12 changed files with 0 additions and 203 deletions
-8
View File
@@ -178,14 +178,6 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int:
return tokenizer.unk_id
def mistral_common_tekkenizer(tokenizer: object) -> "Tekkenizer | None":
"""Return the underlying `Tekkenizer` for a `MistralCommonBackend`."""
mistral = getattr(tokenizer, "tokenizer", None)
instruct = getattr(mistral, "instruct_tokenizer", None)
tekken = getattr(instruct, "tokenizer", None)
return tekken if isinstance(tekken, Tekkenizer) else None
def tekken_convert_ids_to_tokens(
tokenizer: "Tekkenizer", ids: Sequence[int]
) -> list[str | bytes]: