[Rust Frontend] Add HY3 unified parser and local XGrammar structural-tag builder (#53054)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-08-20 19:13:54 -07:00
committed by GitHub
parent c6e19b3be2
commit df3b3422b4
17 changed files with 849 additions and 86 deletions
+33
View File
@@ -149,10 +149,20 @@ fn encode_fastokens_ordinary(
pub struct HuggingFaceTokenizer {
backend: Backend,
special_token_ids: Arc<[u32]>,
added_vocab: Box<[(String, u32)]>,
}
impl HuggingFaceTokenizer {
fn from_hf_backend(tokenizer: HfTokenizer) -> Self {
let added_vocab = {
let mut vocab: Vec<_> = tokenizer
.get_added_tokens_decoder()
.iter()
.map(|(&id, token)| (token.content.clone(), id))
.collect();
vocab.sort_unstable_by_key(|(_, id)| *id);
vocab.into_boxed_slice()
};
let special_token_ids = {
let mut ids: Vec<u32> = tokenizer
.get_added_tokens_decoder()
@@ -167,10 +177,21 @@ impl HuggingFaceTokenizer {
Self {
backend: Backend::Hf(Box::new(tokenizer)),
special_token_ids,
added_vocab,
}
}
fn from_fastokens_backend(tokenizer: FastokensTokenizer) -> Self {
let added_vocab = {
let mut vocab: Vec<_> = tokenizer
.added_tokens()
.into_iter()
.flat_map(|added_tokens| added_tokens.iter())
.map(|token| (token.content.to_string(), token.id))
.collect();
vocab.sort_unstable_by_key(|(_, id)| *id);
vocab.into_boxed_slice()
};
let special_token_ids = {
let mut ids: Vec<u32> = tokenizer
.added_tokens()
@@ -192,6 +213,7 @@ impl HuggingFaceTokenizer {
Self {
backend,
special_token_ids,
added_vocab,
}
}
@@ -292,6 +314,10 @@ impl Tokenizer for HuggingFaceTokenizer {
}
}
fn added_vocab(&self) -> &[(String, u32)] {
&self.added_vocab
}
fn is_special_id(&self, token_id: u32) -> bool {
self.special_token_ids.binary_search(&token_id).is_ok()
}
@@ -477,6 +503,13 @@ mod tests {
tokenizer.encode(SPECIAL_TOKEN, false).unwrap(),
vec![tokenizer.token_to_id(SPECIAL_TOKEN).unwrap()]
);
assert_eq!(
tokenizer.added_vocab(),
vec![
(REGULAR_TOKEN.to_string(), 256),
(SPECIAL_TOKEN.to_string(), 257),
]
);
for text in [
"",
+9
View File
@@ -39,6 +39,15 @@ pub trait Tokenizer: Send + Sync {
/// Convert one token ID into the tokenizer's raw token string.
fn id_to_token(&self, id: u32) -> Option<String>;
/// Borrow tokenizer added-vocabulary entries as `(token, id)` pairs.
///
/// Backends that do not expose the distinction between model vocabulary
/// and added vocabulary return an empty list.
// TODO: add support to all tokenizer backends
fn added_vocab(&self) -> &[(String, u32)] {
&[]
}
/// Return the vocabulary size. Backends that cannot report it fall back to
/// `usize::MAX`, an effectively unbounded value used only by test stubs.
fn vocab_size(&self) -> usize {
+19 -1
View File
@@ -69,6 +69,7 @@ struct TestToken {
pub struct TestTokenizer {
token_to_id: BTreeMap<String, u32>,
id_to_token: BTreeMap<u32, TestToken>,
added_vocab: Vec<(String, u32)>,
unknown_decode: UnknownDecode,
vocab_size: Option<usize>,
bos_token_id: Option<u32>,
@@ -86,6 +87,7 @@ impl TestTokenizer {
Self {
token_to_id: BTreeMap::new(),
id_to_token: BTreeMap::new(),
added_vocab: Vec::new(),
unknown_decode: UnknownDecode::Error,
vocab_size: None,
bos_token_id: None,
@@ -154,9 +156,21 @@ impl TestTokenizer {
if self.token_to_id.insert(token.clone(), id).is_some() {
panic!("configured test token text {token:?} was registered more than once");
}
if self.id_to_token.insert(id, TestToken { text: token, kind }).is_some() {
if self
.id_to_token
.insert(
id,
TestToken {
text: token.clone(),
kind,
},
)
.is_some()
{
panic!("configured test token id {id} was registered more than once");
}
self.added_vocab.push((token, id));
self.added_vocab.sort_unstable_by_key(|(_, id)| *id);
}
fn byte_to_token(id: u32) -> Option<String> {
@@ -254,6 +268,10 @@ impl Tokenizer for TestTokenizer {
.or_else(|| Self::byte_to_token(id))
}
fn added_vocab(&self) -> &[(String, u32)] {
&self.added_vocab
}
fn vocab_size(&self) -> usize {
self.vocab_size.unwrap_or_else(|| self.inferred_vocab_size())
}