[Bugfix] Fix DeepSeek V4/3.2 tokenizer vocab size overcount crashing guided decoding (#51727)
Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
@@ -19,8 +19,6 @@ def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
dsv32_tokenizer = copy.copy(tokenizer)
|
||||
|
||||
added_vocab = tokenizer.get_added_vocab()
|
||||
added_vocab_size = len(added_vocab)
|
||||
tokenizer_vocab_size = tokenizer.vocab_size
|
||||
|
||||
class _DeepseekV32Tokenizer(tokenizer.__class__): # type: ignore
|
||||
def apply_chat_template(
|
||||
@@ -66,10 +64,6 @@ def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
def num_special_tokens_to_add(self) -> int:
|
||||
return len(self.encode(""))
|
||||
|
||||
def __len__(self) -> int:
|
||||
# </think> is an added token in DeepseekV32 tokenizer
|
||||
return tokenizer_vocab_size + added_vocab_size
|
||||
|
||||
def get_added_vocab(self) -> dict[str, int]:
|
||||
return added_vocab.copy()
|
||||
|
||||
|
||||
@@ -19,8 +19,6 @@ def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
dsv4_tokenizer = copy.copy(tokenizer)
|
||||
|
||||
added_vocab = tokenizer.get_added_vocab()
|
||||
added_vocab_size = len(added_vocab)
|
||||
tokenizer_vocab_size = tokenizer.vocab_size
|
||||
|
||||
class _DeepseekV4Tokenizer(tokenizer.__class__): # type: ignore
|
||||
def apply_chat_template(
|
||||
@@ -78,9 +76,6 @@ def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
def num_special_tokens_to_add(self) -> int:
|
||||
return len(self.encode(""))
|
||||
|
||||
def __len__(self) -> int:
|
||||
return tokenizer_vocab_size + added_vocab_size
|
||||
|
||||
def get_added_vocab(self) -> dict[str, int]:
|
||||
return added_vocab.copy()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user