diff --git a/vllm/tokenizers/deepseek_v32.py b/vllm/tokenizers/deepseek_v32.py index b388f05793..0e9aaa2cae 100644 --- a/vllm/tokenizers/deepseek_v32.py +++ b/vllm/tokenizers/deepseek_v32.py @@ -19,8 +19,6 @@ def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: dsv32_tokenizer = copy.copy(tokenizer) added_vocab = tokenizer.get_added_vocab() - added_vocab_size = len(added_vocab) - tokenizer_vocab_size = tokenizer.vocab_size class _DeepseekV32Tokenizer(tokenizer.__class__): # type: ignore def apply_chat_template( @@ -66,10 +64,6 @@ def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: def num_special_tokens_to_add(self) -> int: return len(self.encode("")) - def __len__(self) -> int: - # is an added token in DeepseekV32 tokenizer - return tokenizer_vocab_size + added_vocab_size - def get_added_vocab(self) -> dict[str, int]: return added_vocab.copy() diff --git a/vllm/tokenizers/deepseek_v4.py b/vllm/tokenizers/deepseek_v4.py index ce5b902c31..2a0db215b8 100644 --- a/vllm/tokenizers/deepseek_v4.py +++ b/vllm/tokenizers/deepseek_v4.py @@ -19,8 +19,6 @@ def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: dsv4_tokenizer = copy.copy(tokenizer) added_vocab = tokenizer.get_added_vocab() - added_vocab_size = len(added_vocab) - tokenizer_vocab_size = tokenizer.vocab_size class _DeepseekV4Tokenizer(tokenizer.__class__): # type: ignore def apply_chat_template( @@ -78,9 +76,6 @@ def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: def num_special_tokens_to_add(self) -> int: return len(self.encode("")) - def __len__(self) -> int: - return tokenizer_vocab_size + added_vocab_size - def get_added_vocab(self) -> dict[str, int]: return added_vocab.copy()