[Bugfix][Rust Frontend] Validate out-of-vocab token ids in request params (#44680)

Signed-off-by: Ting Sun <suntcrick@gmail.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Ting SUN
2026-06-11 03:14:11 -07:00
committed by GitHub
co-authored by Bugen Zhao
parent 9492362972
commit 1f9dd7900d
16 changed files with 258 additions and 2 deletions
+7
View File
@@ -174,6 +174,13 @@ impl Tokenizer for HuggingFaceTokenizer {
}
}
fn vocab_size(&self) -> usize {
match &self.backend {
Backend::Hf(t) => t.get_vocab_size(true),
Backend::Fastokens(t) | Backend::FastokensByteLevel(t) => t.vocab_size(),
}
}
fn id_to_token(&self, id: u32) -> Option<String> {
match &self.backend {
Backend::Hf(t) => t.id_to_token(id),
+6
View File
@@ -34,6 +34,12 @@ pub trait Tokenizer: Send + Sync {
None
}
/// Return the vocabulary size. Backends that cannot report it fall back to
/// `usize::MAX`, an effectively unbounded value used only by test stubs.
fn vocab_size(&self) -> usize {
usize::MAX
}
/// Return whether the given token ID is special.
fn is_special_id(&self, _token_id: u32) -> bool {
false
+4
View File
@@ -56,6 +56,10 @@ impl Tokenizer for TekkenTokenizer {
self.inner.id_to_piece(id).ok()
}
fn vocab_size(&self) -> usize {
self.inner.vocab_size()
}
fn is_special_id(&self, token_id: u32) -> bool {
self.inner.is_special_token(token_id)
}
+18
View File
@@ -503,6 +503,13 @@ impl Tokenizer for TiktokenTokenizer {
fn is_special_id(&self, token_id: u32) -> bool {
self.metadata.is_special_id(token_id)
}
fn vocab_size(&self) -> usize {
// Exclusive upper bound on token ids the tokenizer can decode (BPE base
// tokens plus the registered special/reserved slots), used to range-check
// `allowed_token_ids` so tiktoken models are not exempt from validation.
self.metadata.vocab_upper_bound as usize
}
}
/// Select the BPE regex pattern for a tiktoken model based on `config.json`.
@@ -614,6 +621,17 @@ mod tests {
}
}
#[test]
fn tiktoken_vocab_size_reports_upper_bound() {
// The synthetic BPE file has 256 base tokens (bytes 0..=255) and ships no
// sibling config, so the constructor uses the 256-slot reserved fallback,
// giving a vocab upper bound of 512.
let (backends, _dir) = tiktoken_backends();
for backend in backends {
assert_eq!(backend.vocab_size(), 512);
}
}
/// When `config.json` exposes a `vocab_size`, the reserved-token range must
/// be sized to it rather than to the 256-slot fallback. This is the
/// general (non-Kimi-specific) path: any tiktoken model whose own