[Rust Frontend] Add HY3 unified parser and local XGrammar structural-tag builder (#53054)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -149,10 +149,20 @@ fn encode_fastokens_ordinary(
|
||||
pub struct HuggingFaceTokenizer {
|
||||
backend: Backend,
|
||||
special_token_ids: Arc<[u32]>,
|
||||
added_vocab: Box<[(String, u32)]>,
|
||||
}
|
||||
|
||||
impl HuggingFaceTokenizer {
|
||||
fn from_hf_backend(tokenizer: HfTokenizer) -> Self {
|
||||
let added_vocab = {
|
||||
let mut vocab: Vec<_> = tokenizer
|
||||
.get_added_tokens_decoder()
|
||||
.iter()
|
||||
.map(|(&id, token)| (token.content.clone(), id))
|
||||
.collect();
|
||||
vocab.sort_unstable_by_key(|(_, id)| *id);
|
||||
vocab.into_boxed_slice()
|
||||
};
|
||||
let special_token_ids = {
|
||||
let mut ids: Vec<u32> = tokenizer
|
||||
.get_added_tokens_decoder()
|
||||
@@ -167,10 +177,21 @@ impl HuggingFaceTokenizer {
|
||||
Self {
|
||||
backend: Backend::Hf(Box::new(tokenizer)),
|
||||
special_token_ids,
|
||||
added_vocab,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_fastokens_backend(tokenizer: FastokensTokenizer) -> Self {
|
||||
let added_vocab = {
|
||||
let mut vocab: Vec<_> = tokenizer
|
||||
.added_tokens()
|
||||
.into_iter()
|
||||
.flat_map(|added_tokens| added_tokens.iter())
|
||||
.map(|token| (token.content.to_string(), token.id))
|
||||
.collect();
|
||||
vocab.sort_unstable_by_key(|(_, id)| *id);
|
||||
vocab.into_boxed_slice()
|
||||
};
|
||||
let special_token_ids = {
|
||||
let mut ids: Vec<u32> = tokenizer
|
||||
.added_tokens()
|
||||
@@ -192,6 +213,7 @@ impl HuggingFaceTokenizer {
|
||||
Self {
|
||||
backend,
|
||||
special_token_ids,
|
||||
added_vocab,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -292,6 +314,10 @@ impl Tokenizer for HuggingFaceTokenizer {
|
||||
}
|
||||
}
|
||||
|
||||
fn added_vocab(&self) -> &[(String, u32)] {
|
||||
&self.added_vocab
|
||||
}
|
||||
|
||||
fn is_special_id(&self, token_id: u32) -> bool {
|
||||
self.special_token_ids.binary_search(&token_id).is_ok()
|
||||
}
|
||||
@@ -477,6 +503,13 @@ mod tests {
|
||||
tokenizer.encode(SPECIAL_TOKEN, false).unwrap(),
|
||||
vec![tokenizer.token_to_id(SPECIAL_TOKEN).unwrap()]
|
||||
);
|
||||
assert_eq!(
|
||||
tokenizer.added_vocab(),
|
||||
vec![
|
||||
(REGULAR_TOKEN.to_string(), 256),
|
||||
(SPECIAL_TOKEN.to_string(), 257),
|
||||
]
|
||||
);
|
||||
|
||||
for text in [
|
||||
"",
|
||||
|
||||
@@ -39,6 +39,15 @@ pub trait Tokenizer: Send + Sync {
|
||||
/// Convert one token ID into the tokenizer's raw token string.
|
||||
fn id_to_token(&self, id: u32) -> Option<String>;
|
||||
|
||||
/// Borrow tokenizer added-vocabulary entries as `(token, id)` pairs.
|
||||
///
|
||||
/// Backends that do not expose the distinction between model vocabulary
|
||||
/// and added vocabulary return an empty list.
|
||||
// TODO: add support to all tokenizer backends
|
||||
fn added_vocab(&self) -> &[(String, u32)] {
|
||||
&[]
|
||||
}
|
||||
|
||||
/// Return the vocabulary size. Backends that cannot report it fall back to
|
||||
/// `usize::MAX`, an effectively unbounded value used only by test stubs.
|
||||
fn vocab_size(&self) -> usize {
|
||||
|
||||
@@ -69,6 +69,7 @@ struct TestToken {
|
||||
pub struct TestTokenizer {
|
||||
token_to_id: BTreeMap<String, u32>,
|
||||
id_to_token: BTreeMap<u32, TestToken>,
|
||||
added_vocab: Vec<(String, u32)>,
|
||||
unknown_decode: UnknownDecode,
|
||||
vocab_size: Option<usize>,
|
||||
bos_token_id: Option<u32>,
|
||||
@@ -86,6 +87,7 @@ impl TestTokenizer {
|
||||
Self {
|
||||
token_to_id: BTreeMap::new(),
|
||||
id_to_token: BTreeMap::new(),
|
||||
added_vocab: Vec::new(),
|
||||
unknown_decode: UnknownDecode::Error,
|
||||
vocab_size: None,
|
||||
bos_token_id: None,
|
||||
@@ -154,9 +156,21 @@ impl TestTokenizer {
|
||||
if self.token_to_id.insert(token.clone(), id).is_some() {
|
||||
panic!("configured test token text {token:?} was registered more than once");
|
||||
}
|
||||
if self.id_to_token.insert(id, TestToken { text: token, kind }).is_some() {
|
||||
if self
|
||||
.id_to_token
|
||||
.insert(
|
||||
id,
|
||||
TestToken {
|
||||
text: token.clone(),
|
||||
kind,
|
||||
},
|
||||
)
|
||||
.is_some()
|
||||
{
|
||||
panic!("configured test token id {id} was registered more than once");
|
||||
}
|
||||
self.added_vocab.push((token, id));
|
||||
self.added_vocab.sort_unstable_by_key(|(_, id)| *id);
|
||||
}
|
||||
|
||||
fn byte_to_token(id: u32) -> Option<String> {
|
||||
@@ -254,6 +268,10 @@ impl Tokenizer for TestTokenizer {
|
||||
.or_else(|| Self::byte_to_token(id))
|
||||
}
|
||||
|
||||
fn added_vocab(&self) -> &[(String, u32)] {
|
||||
&self.added_vocab
|
||||
}
|
||||
|
||||
fn vocab_size(&self) -> usize {
|
||||
self.vocab_size.unwrap_or_else(|| self.inferred_vocab_size())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user