[Bugfix][Rust Frontend] Fix missing added tokens in hf/fastokens tokenizer (#44683)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>

rust/src/chat/src/multimodal.rs

@@ -225,7 +225,7 @@ impl MultimodalModelInfo {
     ///
     /// The HF renderer uses this token while flattening image content in string
     /// content format.
-    pub(crate) fn placeholder_token(&self) -> &str {
+    pub fn placeholder_token(&self) -> &str {
         &self.spec.placeholder_token
     }
 }

rust/src/server/src/routes/tests.rs

@@ -427,6 +427,11 @@ impl Tokenizer for FakeChatTokenizer {
                 rest = stripped;
                 continue;
             }
+            if let Some(stripped) = rest.strip_prefix("<|image_pad|>") {
+                token_ids.push(151655);
+                rest = stripped;
+                continue;
+            }
 
             let ch = rest.chars().next().expect("rest is not empty");
             let mut buf = [0; 4];
@@ -547,11 +552,16 @@ impl ChatBackend for FakeChatBackend {
 
 impl ChatRenderer for FakeChatBackend {
     fn render(&self, request: &ChatRequest) -> vllm_chat::Result<vllm_chat::RenderedPrompt> {
+        let placeholder = self
+            .multimodal_model_info
+            .as_ref()
+            .map(|info| info.placeholder_token())
+            .unwrap_or("<image>");
         let mut prompt = String::new();
         for message in &request.messages {
             prompt.push_str(message.role().as_str());
             prompt.push_str(": ");
-            prompt.push_str(&render_fake_message_content(message)?);
+            prompt.push_str(&render_fake_message_content(message, placeholder)?);
             prompt.push('\n');
         }
         if request.chat_options.add_generation_prompt() {
@@ -563,17 +573,20 @@ impl ChatRenderer for FakeChatBackend {
     }
 }
 
-fn render_fake_message_content(message: &ChatMessage) -> vllm_chat::Result<String> {
+fn render_fake_message_content(
+    message: &ChatMessage,
+    placeholder: &str,
+) -> vllm_chat::Result<String> {
     match message {
         ChatMessage::System { content }
         | ChatMessage::Developer { content, .. }
         | ChatMessage::User { content }
-        | ChatMessage::ToolResponse { content, .. } => render_fake_content(content),
+        | ChatMessage::ToolResponse { content, .. } => render_fake_content(content, placeholder),
         ChatMessage::Assistant { .. } => message.text_content(),
     }
 }
 
-fn render_fake_content(content: &ChatContent) -> vllm_chat::Result<String> {
+fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::Result<String> {
     Ok(match content {
         ChatContent::Text(text) => text.clone(),
         ChatContent::Parts(parts) => {
@@ -581,7 +594,7 @@ fn render_fake_content(content: &ChatContent) -> vllm_chat::Result<String> {
             for part in parts {
                 match part {
                     ChatContentPart::Text { text } => out.push_str(text),
-                    ChatContentPart::ImageUrl { .. } => out.push_str("<image>"),
+                    ChatContentPart::ImageUrl { .. } => out.push_str(placeholder),
                 }
             }
             out

rust/src/tokenizer/src/hf.rs

@@ -8,8 +8,11 @@ use tokenizers::Tokenizer as HfTokenizer;
 use tracing::{info, warn};
 
 use crate::byte_level_decode::decode_byte_level;
+use crate::hf::added_tokens::load_tokenizer_json_with_extra_tokens;
 use crate::{Result, Tokenizer};
 
+mod added_tokens;
+
 enum Backend {
     Hf(Box<HfTokenizer>),
     Fastokens(Box<FastokensTokenizer>),
@@ -104,7 +107,8 @@ impl HuggingFaceTokenizer {
     /// Load from `tokenizer.json` with `fastokens`.
     pub fn new_fastokens(path: &Path) -> Result<Self> {
         info!(path = %path.display(), "loading tokenizer with fastokens");
-        let t = FastokensTokenizer::from_file(path)
+        let tokenizer_json = load_tokenizer_json_with_extra_tokens(path)?;
+        let t = FastokensTokenizer::from_json(tokenizer_json)
             .map_err(|error| tokenizer_error!("failed to load tokenizer: {}", error.as_report()))?;
         Ok(Self::from_fastokens_backend(t))
     }
@@ -112,7 +116,8 @@ impl HuggingFaceTokenizer {
     /// Load from `tokenizer.json` with Hugging Face `tokenizers`.
     pub fn new_hf(path: &Path) -> Result<Self> {
         info!(path = %path.display(), "loading tokenizer with huggingface tokenizers");
-        let t = HfTokenizer::from_file(path)
+        let tokenizer_json = load_tokenizer_json_with_extra_tokens(path)?;
+        let t = serde_json::from_value::<HfTokenizer>(tokenizer_json)
             .map_err(|error| tokenizer_error!("failed to load tokenizer: {}", error.as_report()))?;
         Ok(Self::from_hf_backend(t))
     }
@@ -250,6 +255,37 @@ mod tests {
         assert!(wrapper.is_special_id(special_id));
     }
 
+    #[test]
+    fn constructors_merge_extra_added_tokens_from_tokenizer_config() {
+        let tokenizer = tiny_bpe_tokenizer();
+
+        let dir = tempdir().expect("create temp dir");
+        let path = dir.path().join("tokenizer.json");
+        tokenizer.save(&path, false).expect("save tokenizer json");
+        std::fs::write(
+            dir.path().join("tokenizer_config.json"),
+            r#"{
+                "added_tokens_decoder": {
+                    "9": {
+                        "content": "<|image_pad|>",
+                        "special": true,
+                        "normalized": false
+                    }
+                }
+            }"#,
+        )
+        .expect("write tokenizer config");
+
+        for wrapper in [
+            HuggingFaceTokenizer::new_fastokens(&path).expect("load fastokens wrapper"),
+            HuggingFaceTokenizer::new_hf(&path).expect("load hf wrapper"),
+        ] {
+            assert_eq!(wrapper.token_to_id("<|image_pad|>"), Some(9));
+            assert_eq!(wrapper.id_to_token(9).as_deref(), Some("<|image_pad|>"));
+            assert!(wrapper.is_special_id(9));
+        }
+    }
+
     /// BPE tokenizer that round-trips through fastokens with a genuine
     /// `ByteLevel` decoder; vocab covers both GPT-2 (Ġ U+0120) and non-GPT-2
     /// (｜ U+FF5C) codepoints.

rust/src/tokenizer/src/hf/added_tokens.rs

@@ -0,0 +1,158 @@
+use serde::{Deserialize, Serialize};
+use thiserror_ext::AsReport as _;
+use tracing::warn;
+
+use crate::Result;
+
+use std::{fs, path::Path};
+
+/// Minimal `tokenizer.json` projection used to patch `added_tokens` while
+/// preserving the rest of the tokenizer definition verbatim.
+#[derive(Debug, Deserialize, Serialize)]
+struct TokenizerJson {
+    #[serde(default)]
+    added_tokens: Vec<AddedTokenConfig>,
+    #[serde(flatten)]
+    extra: serde_json::Map<String, serde_json::Value>,
+}
+
+/// Minimal `tokenizer_config.json` projection for Hugging Face's
+/// `added_tokens_decoder` map. Other config keys are intentionally ignored.
+#[derive(Debug, Deserialize)]
+struct TokenizerConfigJson {
+    #[serde(default)]
+    added_tokens_decoder: std::collections::HashMap<String, AddedTokenConfig>,
+}
+
+/// Hugging Face added-token payload. `tokenizer.json` stores `id` inside each
+/// item, while `tokenizer_config.json` stores it as the map key.
+#[derive(Clone, Debug, Deserialize, Serialize)]
+struct AddedTokenConfig {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    id: Option<u32>,
+    content: String,
+    #[serde(default)]
+    single_word: bool,
+    #[serde(default)]
+    lstrip: bool,
+    #[serde(default)]
+    rstrip: bool,
+    #[serde(default)]
+    normalized: bool,
+    #[serde(default)]
+    special: bool,
+}
+
+impl AddedTokenConfig {
+    /// Return this added-token payload in `tokenizer.json` shape by filling the
+    /// numeric token id that came from `added_tokens_decoder`'s string key.
+    fn with_id(mut self, id: u32) -> Self {
+        self.id = Some(id);
+        self
+    }
+}
+
+/// Read `tokenizer.json`, then merge in extra added tokens from `tokenizer_config.json`.
+pub(super) fn load_tokenizer_json_with_extra_tokens(path: &Path) -> Result<serde_json::Value> {
+    let tokenizer_json = fs::read_to_string(path)
+        .map_err(|error| tokenizer_error!("failed to read {}: {}", path.display(), error))?;
+    let mut tokenizer_json: TokenizerJson = serde_json::from_str(&tokenizer_json)
+        .map_err(|error| tokenizer_error!("failed to parse {}: {}", path.display(), error))?;
+
+    if let Some(parent) = path.parent() {
+        let config_path = parent.join("tokenizer_config.json");
+        if config_path.exists() {
+            match load_tokenizer_config_json(&config_path) {
+                Ok(config_json) => merge_added_tokens_from_config(&mut tokenizer_json, config_json),
+                Err(error) => {
+                    warn!(
+                        path = %config_path.display(),
+                        error = %error.as_report(),
+                        "failed to load tokenizer_config.json; skipping extra added tokens"
+                    );
+                }
+            }
+        }
+    }
+
+    serde_json::to_value(tokenizer_json)
+        .map_err(|error| tokenizer_error!("failed to serialize tokenizer json: {}", error))
+}
+
+/// Read and parse a sibling `tokenizer_config.json`.
+fn load_tokenizer_config_json(path: &Path) -> Result<TokenizerConfigJson> {
+    let text = fs::read_to_string(path)
+        .map_err(|error| tokenizer_error!("failed to read {}: {}", path.display(), error))?;
+    serde_json::from_str(&text)
+        .map_err(|error| tokenizer_error!("failed to parse {}: {}", path.display(), error))
+}
+
+/// Merge added_tokens in `tokenizer.json` and `tokenizer_config.json`.
+fn merge_added_tokens_from_config(
+    tokenizer_json: &mut TokenizerJson,
+    config_json: TokenizerConfigJson,
+) {
+    use std::collections::HashSet;
+
+    let mut existing_ids: HashSet<u32> =
+        tokenizer_json.added_tokens.iter().filter_map(|token| token.id).collect();
+
+    let mut extra_tokens = Vec::with_capacity(config_json.added_tokens_decoder.len());
+    for (id_str, token) in config_json.added_tokens_decoder {
+        let id = match id_str.parse::<u32>() {
+            Ok(id) => id,
+            Err(_) => continue,
+        };
+        extra_tokens.push((id, token));
+    }
+    extra_tokens.sort_unstable_by_key(|(id, _)| *id);
+
+    for (id, token) in extra_tokens {
+        if existing_ids.contains(&id) {
+            continue;
+        }
+
+        // Convert from decoder format to added_tokens array format by adding the "id" field.
+        tokenizer_json.added_tokens.push(token.with_id(id));
+        existing_ids.insert(id);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn merge_added_tokens_from_config_preserves_unmodeled_fields() {
+        let mut tokenizer_json: TokenizerJson = serde_json::from_value(serde_json::json!({
+            "version": "1.0",
+            "added_tokens": [
+                {"id": 0, "content": "<unk>", "special": true}
+            ],
+            "model": {"type": "WordLevel"}
+        }))
+        .expect("parse tokenizer json");
+
+        let config_json: TokenizerConfigJson = serde_json::from_value(serde_json::json!({
+            "chat_template": "{{ messages }}",
+            "added_tokens_decoder": {
+                "1": {
+                    "content": "<|image_pad|>",
+                    "special": true,
+                    "normalized": false
+                }
+            }
+        }))
+        .expect("parse tokenizer config");
+
+        merge_added_tokens_from_config(&mut tokenizer_json, config_json);
+        let merged = serde_json::to_value(tokenizer_json).expect("serialize tokenizer json");
+
+        assert_eq!(merged["version"], "1.0");
+        assert_eq!(merged["model"]["type"], "WordLevel");
+        assert_eq!(merged["added_tokens"][1]["id"], 1);
+        assert_eq!(merged["added_tokens"][1]["content"], "<|image_pad|>");
+        assert_eq!(merged["added_tokens"][1]["special"], true);
+        assert_eq!(merged["added_tokens"][1]["normalized"], false);
+    }
+}