feat(frontend): early-fail tokenization guard for user requests (#31366)
Signed-off-by: limingliang <limingliang@stepfun.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: limingliang <limingliang@stepfun.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
limingliang
DarkLight1337
parent
20d7454c9b
commit
a32cb49b60
@@ -115,6 +115,10 @@ class DeepseekV32Tokenizer(CachedHfTokenizer):
|
||||
def max_token_id(self) -> int:
|
||||
return self.tokenizer.max_token_id
|
||||
|
||||
@property
|
||||
def max_chars_per_token(self) -> int:
|
||||
return self.tokenizer.max_chars_per_token
|
||||
|
||||
@property
|
||||
def truncation_side(self) -> str:
|
||||
return self.tokenizer.truncation_side
|
||||
|
||||
@@ -277,6 +277,8 @@ class Grok2Tokenizer(TokenizerLike):
|
||||
self._pad_token_id = self._special_tokens.get(PAD, self._eos_token_id)
|
||||
self._unk_token_id = self._pad_token_id
|
||||
|
||||
self._max_chars_per_token = max(len(tok) for tok in self._token_to_id)
|
||||
|
||||
def num_special_tokens_to_add(self) -> int:
|
||||
return 0
|
||||
|
||||
@@ -312,6 +314,10 @@ class Grok2Tokenizer(TokenizerLike):
|
||||
def max_token_id(self) -> int:
|
||||
return self._tokenizer.n_vocab - 1
|
||||
|
||||
@property
|
||||
def max_chars_per_token(self) -> int:
|
||||
return self._max_chars_per_token
|
||||
|
||||
@property
|
||||
def truncation_side(self) -> str:
|
||||
return self._truncation_side
|
||||
|
||||
@@ -28,6 +28,8 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
tokenizer_len = len(tokenizer)
|
||||
|
||||
max_token_id = max(tokenizer_vocab.values())
|
||||
max_chars_per_token = max(len(tok) for tok in tokenizer_vocab)
|
||||
|
||||
# Some tokenizers (e.g., QwenTokenizer) have special tokens that
|
||||
# are added and included in the implementation of the vocab_size
|
||||
# property, but not in get_vocab(); if there is an implementation
|
||||
@@ -49,6 +51,10 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
def max_token_id(self) -> int:
|
||||
return max_token_id
|
||||
|
||||
@property
|
||||
def max_chars_per_token(self) -> int:
|
||||
return max_chars_per_token
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return tokenizer_vocab
|
||||
|
||||
|
||||
@@ -272,6 +272,7 @@ class MistralTokenizer(TokenizerLike):
|
||||
# Vocab sorted by token id.
|
||||
self._vocab = self.tokenizer.vocab()
|
||||
self._max_token_id = self.vocab_size - 1
|
||||
self._max_chars_per_token = max(len(tok) for tok in self._vocab)
|
||||
|
||||
# Cache special tokens for faster access.
|
||||
self._special_token_ids = self._get_special_token_ids()
|
||||
@@ -325,6 +326,10 @@ class MistralTokenizer(TokenizerLike):
|
||||
def max_token_id(self) -> int:
|
||||
return self._max_token_id
|
||||
|
||||
@property
|
||||
def max_chars_per_token(self) -> int:
|
||||
return self._max_chars_per_token
|
||||
|
||||
@property
|
||||
def truncation_side(self) -> str:
|
||||
return self.transformers_tokenizer.truncation_side
|
||||
|
||||
@@ -57,6 +57,10 @@ class TokenizerLike(Protocol):
|
||||
def max_token_id(self) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def max_chars_per_token(self) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def truncation_side(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
Reference in New Issue
Block a user