[Bugfix][Rust Frontend] Validate out-of-vocab token ids in request params (#44680)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -174,6 +174,13 @@ impl Tokenizer for HuggingFaceTokenizer {
|
||||
}
|
||||
}
|
||||
|
||||
fn vocab_size(&self) -> usize {
|
||||
match &self.backend {
|
||||
Backend::Hf(t) => t.get_vocab_size(true),
|
||||
Backend::Fastokens(t) | Backend::FastokensByteLevel(t) => t.vocab_size(),
|
||||
}
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
match &self.backend {
|
||||
Backend::Hf(t) => t.id_to_token(id),
|
||||
|
||||
@@ -34,6 +34,12 @@ pub trait Tokenizer: Send + Sync {
|
||||
None
|
||||
}
|
||||
|
||||
/// Return the vocabulary size. Backends that cannot report it fall back to
|
||||
/// `usize::MAX`, an effectively unbounded value used only by test stubs.
|
||||
fn vocab_size(&self) -> usize {
|
||||
usize::MAX
|
||||
}
|
||||
|
||||
/// Return whether the given token ID is special.
|
||||
fn is_special_id(&self, _token_id: u32) -> bool {
|
||||
false
|
||||
|
||||
@@ -56,6 +56,10 @@ impl Tokenizer for TekkenTokenizer {
|
||||
self.inner.id_to_piece(id).ok()
|
||||
}
|
||||
|
||||
fn vocab_size(&self) -> usize {
|
||||
self.inner.vocab_size()
|
||||
}
|
||||
|
||||
fn is_special_id(&self, token_id: u32) -> bool {
|
||||
self.inner.is_special_token(token_id)
|
||||
}
|
||||
|
||||
@@ -503,6 +503,13 @@ impl Tokenizer for TiktokenTokenizer {
|
||||
fn is_special_id(&self, token_id: u32) -> bool {
|
||||
self.metadata.is_special_id(token_id)
|
||||
}
|
||||
|
||||
fn vocab_size(&self) -> usize {
|
||||
// Exclusive upper bound on token ids the tokenizer can decode (BPE base
|
||||
// tokens plus the registered special/reserved slots), used to range-check
|
||||
// `allowed_token_ids` so tiktoken models are not exempt from validation.
|
||||
self.metadata.vocab_upper_bound as usize
|
||||
}
|
||||
}
|
||||
|
||||
/// Select the BPE regex pattern for a tiktoken model based on `config.json`.
|
||||
@@ -614,6 +621,17 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiktoken_vocab_size_reports_upper_bound() {
|
||||
// The synthetic BPE file has 256 base tokens (bytes 0..=255) and ships no
|
||||
// sibling config, so the constructor uses the 256-slot reserved fallback,
|
||||
// giving a vocab upper bound of 512.
|
||||
let (backends, _dir) = tiktoken_backends();
|
||||
for backend in backends {
|
||||
assert_eq!(backend.vocab_size(), 512);
|
||||
}
|
||||
}
|
||||
|
||||
/// When `config.json` exposes a `vocab_size`, the reserved-token range must
|
||||
/// be sized to it rather than to the 256-slot fallback. This is the
|
||||
/// general (non-Kimi-specific) path: any tiktoken model whose own
|
||||
|
||||
Reference in New Issue
Block a user