[Bugfix][Rust Frontend] Fix missing added tokens in hf/fastokens tokenizer (#44683)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Isotr0py
2026-06-10 03:52:41 -07:00
committed by GitHub
co-authored by Bugen Zhao
parent a1ec011a83
commit 9ad08c4d15
4 changed files with 215 additions and 8 deletions
+18 -5
View File
@@ -427,6 +427,11 @@ impl Tokenizer for FakeChatTokenizer {
rest = stripped;
continue;
}
if let Some(stripped) = rest.strip_prefix("<|image_pad|>") {
token_ids.push(151655);
rest = stripped;
continue;
}
let ch = rest.chars().next().expect("rest is not empty");
let mut buf = [0; 4];
@@ -547,11 +552,16 @@ impl ChatBackend for FakeChatBackend {
impl ChatRenderer for FakeChatBackend {
fn render(&self, request: &ChatRequest) -> vllm_chat::Result<vllm_chat::RenderedPrompt> {
let placeholder = self
.multimodal_model_info
.as_ref()
.map(|info| info.placeholder_token())
.unwrap_or("<image>");
let mut prompt = String::new();
for message in &request.messages {
prompt.push_str(message.role().as_str());
prompt.push_str(": ");
prompt.push_str(&render_fake_message_content(message)?);
prompt.push_str(&render_fake_message_content(message, placeholder)?);
prompt.push('\n');
}
if request.chat_options.add_generation_prompt() {
@@ -563,17 +573,20 @@ impl ChatRenderer for FakeChatBackend {
}
}
fn render_fake_message_content(message: &ChatMessage) -> vllm_chat::Result<String> {
fn render_fake_message_content(
message: &ChatMessage,
placeholder: &str,
) -> vllm_chat::Result<String> {
match message {
ChatMessage::System { content }
| ChatMessage::Developer { content, .. }
| ChatMessage::User { content }
| ChatMessage::ToolResponse { content, .. } => render_fake_content(content),
| ChatMessage::ToolResponse { content, .. } => render_fake_content(content, placeholder),
ChatMessage::Assistant { .. } => message.text_content(),
}
}
fn render_fake_content(content: &ChatContent) -> vllm_chat::Result<String> {
fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::Result<String> {
Ok(match content {
ChatContent::Text(text) => text.clone(),
ChatContent::Parts(parts) => {
@@ -581,7 +594,7 @@ fn render_fake_content(content: &ChatContent) -> vllm_chat::Result<String> {
for part in parts {
match part {
ChatContentPart::Text { text } => out.push_str(text),
ChatContentPart::ImageUrl { .. } => out.push_str("<image>"),
ChatContentPart::ImageUrl { .. } => out.push_str(placeholder),
}
}
out