[Bugfix] Fix DeepSeek V4/3.2 tokenizer vocab size overcount crashing guided decoding (#51727)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Flora Feng
2026-08-10 23:00:08 +00:00
committed by GitHub
co-authored by Wentao Ye
parent 3e174bb73c
commit 8bcc916a98
2 changed files with 0 additions and 11 deletions
-6
View File
@@ -19,8 +19,6 @@ def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
dsv32_tokenizer = copy.copy(tokenizer)
added_vocab = tokenizer.get_added_vocab()
added_vocab_size = len(added_vocab)
tokenizer_vocab_size = tokenizer.vocab_size
class _DeepseekV32Tokenizer(tokenizer.__class__): # type: ignore
def apply_chat_template(
@@ -66,10 +64,6 @@ def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
def num_special_tokens_to_add(self) -> int:
return len(self.encode(""))
def __len__(self) -> int:
# </think> is an added token in DeepseekV32 tokenizer
return tokenizer_vocab_size + added_vocab_size
def get_added_vocab(self) -> dict[str, int]:
return added_vocab.copy()
-5
View File
@@ -19,8 +19,6 @@ def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
dsv4_tokenizer = copy.copy(tokenizer)
added_vocab = tokenizer.get_added_vocab()
added_vocab_size = len(added_vocab)
tokenizer_vocab_size = tokenizer.vocab_size
class _DeepseekV4Tokenizer(tokenizer.__class__): # type: ignore
def apply_chat_template(
@@ -78,9 +76,6 @@ def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
def num_special_tokens_to_add(self) -> int:
return len(self.encode(""))
def __len__(self) -> int:
return tokenizer_vocab_size + added_vocab_size
def get_added_vocab(self) -> dict[str, int]:
return added_vocab.copy()