[Model] Add Inkling model support [1/N] (#48799)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Woosuk Kwon
2026-07-15 23:40:07 -07:00
committed by GitHub
co-authored by Bugen Zhao Giancarlo Delfin Isotr0py Isotr0py Jee Jee Li Roger Wang Yifan Qiao Claude Fable 5 OpenAI Codex
parent 8bfd683901
commit 6570c9800c
95 changed files with 12137 additions and 62 deletions
+2
View File
@@ -20,6 +20,7 @@ use crate::output::{
use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
use crate::renderer::{
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer,
InklingChatRenderer,
};
use crate::request::ChatRequest;
use crate::{DynChatOutputProcessor, RendererSelection};
@@ -71,6 +72,7 @@ impl HfChatBackend {
RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()),
RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
};
info!(
+3 -3
View File
@@ -33,7 +33,7 @@ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
pub use renderer::hf::ChatTemplateContentFormatOption;
pub use renderer::{
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
HarmonyChatRenderer, RenderedPrompt, RendererSelection,
HarmonyChatRenderer, InklingChatRenderer, RenderedPrompt, RendererSelection,
};
pub use request::{
ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
@@ -294,7 +294,7 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string());
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string());
}
#[test]
@@ -305,6 +305,6 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
}
}
+112
View File
@@ -79,6 +79,38 @@ mod tests {
use super::*;
const AUDIO_PAD_ID: u32 = 151_676;
const INKLING_AUDIO_MARKER_ID: u32 = 200_020;
const INKLING_AUDIO_EMBED_ID: i32 = 200_053;
fn inkling_info(decoder_dmodel: serde_json::Value) -> MultimodalModelInfo {
let config = serde_json::json!({
"model_type": "inkling_mm_model",
"audio_config": {
"decoder_dmodel": decoder_dmodel,
"n_mel_bins": 80,
"mel_vocab_size": 16,
"dmel_min_value": -7.0,
"dmel_max_value": 2.0
}
});
let tokenizer = TestTokenizer::new()
.with_regular_token("<|content_image|>", 200_005)
.with_regular_token("<|content_audio_input|>", INKLING_AUDIO_MARKER_ID);
let context = MultimodalModelContext {
model_id: "inkling-test".to_string(),
model_type: Some("inkling_mm_model".to_string()),
config,
tokenizer: TokenizerResolver(Arc::new(tokenizer)),
};
MultimodalModelInfo::from_loaded(
context,
PreProcessorConfig::default(),
PreProcessorConfig::default(),
)
.unwrap()
.expect("Inkling multimodal support")
}
fn qwen3_asr_info() -> MultimodalModelInfo {
let context = MultimodalModelContext {
@@ -120,6 +152,40 @@ mod tests {
bytes
}
#[test]
fn resolves_inkling_audio_from_model_spec() {
let info = inkling_info(serde_json::json!(1024));
let support = info.audio.as_ref().expect("audio support");
assert_eq!(
info.placeholder_token(Modality::Audio),
Some("<|content_audio_input|>")
);
assert_eq!(support.placeholder.marker_token_id, INKLING_AUDIO_MARKER_ID);
assert_eq!(
support.placeholder.embed_token_id,
INKLING_AUDIO_EMBED_ID as u32
);
assert_eq!(support.spec.primary_key(), AUDIO_PRIMARY_KEY);
assert!(matches!(
&support.spec.field_layouts.encoder_input,
llm_multimodal::FieldLayout::Flat { sizes_key }
if sizes_key == "num_audio_tokens"
));
assert!(matches!(
support.spec.field_layouts.model_specific.get("num_audio_tokens"),
Some(llm_multimodal::FieldLayout::Batched)
));
}
#[test]
fn inkling_decoder_config_gates_audio_capability() {
let info = inkling_info(serde_json::Value::Null);
assert!(info.audio.is_none());
assert_eq!(info.placeholder_token(Modality::Audio), None);
}
#[test]
fn resolves_qwen_audio_from_model_spec() {
let info = qwen3_asr_info();
@@ -210,4 +276,50 @@ mod tests {
if tensor.dtype == "int64" && tensor.shape.is_empty()
));
}
#[tokio::test]
async fn inkling_tracker_and_processor_use_standard_audio_key() {
let info = inkling_info(serde_json::json!(1024));
let wav = wav_i16_mono(16_000, &[0; 1_600]);
let expected_hash = llm_multimodal::hasher::hash_audio(&wav);
let fetched = info
.fetch_media(vec![MediaContentPart::AudioData {
data: wav,
mime_type: Some("audio/wav".to_string()),
uuid: Some("audio-1".to_string()),
}])
.await
.unwrap();
let prepared = info.prepare_audios(fetched.audios, fetched.audio_uuids).await.unwrap();
assert_eq!(prepared.replacements.len(), 1);
assert_eq!(
prepared.replacements[0].tokens[0],
INKLING_AUDIO_MARKER_ID as i32
);
assert!(
prepared.replacements[0].tokens[1..]
.iter()
.all(|token| *token == INKLING_AUDIO_EMBED_ID)
);
let item = &prepared.items[0];
assert_eq!(item.hash, expected_hash);
assert_eq!(item.uuid.as_deref(), Some("audio-1"));
let features = &item.data[AUDIO_PRIMARY_KEY];
assert!(matches!(&features.field, MmField::Flat(_)));
assert!(matches!(
features.data.as_ref(),
Some(MmKwargValue::Tensor(tensor))
if tensor.dtype == "float32" && tensor.shape.get(1) == Some(&80)
));
let count = &item.data["num_audio_tokens"];
assert!(matches!(&count.field, MmField::Batched(_)));
assert!(matches!(
count.data.as_ref(),
Some(MmKwargValue::Tensor(tensor))
if tensor.dtype == "int64" && tensor.shape.is_empty()
));
}
}
@@ -23,6 +23,7 @@ pub mod names {
pub const DEEPSEEK_V3: &str = "deepseek_v3";
pub const DEEPSEEK_V4: &str = "deepseek_v4";
pub const GEMMA4: &str = "gemma4";
pub const INKLING: &str = "inkling";
pub const GLM45: &str = "glm45";
pub const KIMI: &str = "kimi";
pub const KIMI_K2: &str = "kimi_k2";
@@ -63,6 +64,7 @@ impl ReasoningParserFactory {
.register_parser::<DeepSeekV3ReasoningParser>(names::DEEPSEEK_V3)
.register_parser::<DeepSeekV4ReasoningParser>(names::DEEPSEEK_V4)
.register_unified_dummy(names::GEMMA4)
.register_unified_dummy(names::INKLING)
.register_parser::<Glm45ReasoningParser>(names::GLM45)
.register_parser::<KimiReasoningParser>(names::KIMI)
.register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
+2
View File
@@ -25,6 +25,7 @@ pub mod names {
pub const GLM45: &str = "glm45";
pub const GLM47: &str = "glm47";
pub const GEMMA4: &str = "gemma4";
pub const INKLING: &str = "inkling";
pub const GRANITE4: &str = "granite4";
pub const HERMES: &str = "hermes";
pub const HY_V3: &str = "hy_v3";
@@ -70,6 +71,7 @@ impl ToolParserFactory {
.register_parser::<Glm45MoeToolParser>(names::GLM45)
.register_parser::<Glm47MoeToolParser>(names::GLM47)
.register_unified_dummy(names::GEMMA4)
.register_unified_dummy(names::INKLING)
.register_parser::<Granite4ToolParser>(names::GRANITE4)
.register_parser::<HermesToolParser>(names::HERMES)
.register_parser::<HyV3ToolParser>(names::HY_V3)
+24 -2
View File
@@ -5,7 +5,7 @@
use std::sync::LazyLock;
pub use vllm_parser::unified::{Gemma4UnifiedParser, UnifiedParser};
pub use vllm_parser::unified::{Gemma4UnifiedParser, InklingUnifiedParser, UnifiedParser};
use vllm_tokenizer::DynTokenizer;
use crate::parser::ParserFactory;
@@ -14,6 +14,7 @@ use crate::request::ChatTool;
/// Canonical public names for registered unified parsers.
pub mod names {
pub const GEMMA4: &str = "gemma4";
pub const INKLING: &str = "inkling";
}
/// Constructor signature for one registered unified parser implementation.
@@ -37,10 +38,12 @@ impl UnifiedParserFactory {
let mut factory = Self::default();
factory.register_parser::<Gemma4UnifiedParser>(names::GEMMA4);
factory.register_parser::<InklingUnifiedParser>(names::INKLING);
factory
.register_pattern("gemma-4", names::GEMMA4)
.register_pattern("gemma4", names::GEMMA4);
.register_pattern("gemma4", names::GEMMA4)
.register_pattern("inkling", names::INKLING);
factory
}
@@ -88,6 +91,13 @@ mod tests {
.with_regular_token("<channel|>", 257)
}
fn inkling_tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_regular_token("<|message_model|>", 200001)
.with_regular_token("<|content_text|>", 200004)
.with_regular_token("<|content_thinking|>", 200008)
}
#[test]
fn factory_registers_gemma4() {
let factory = UnifiedParserFactory::new();
@@ -99,4 +109,16 @@ mod tests {
);
factory.create(names::GEMMA4, &[], Arc::new(tokenizer())).unwrap();
}
#[test]
fn factory_registers_inkling() {
let factory = UnifiedParserFactory::new();
assert!(factory.contains(names::INKLING));
assert_eq!(
factory.resolve_name_for_model("thinkingmachines/Inkling"),
Some(names::INKLING)
);
factory.create(names::INKLING, &[], Arc::new(inkling_tokenizer())).unwrap();
}
}
@@ -0,0 +1,21 @@
{
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "transcribe"
},
{
"type": "input_audio",
"data": ""
},
{
"type": "audio_url",
"audio_url": "data:audio/wav;base64,"
}
]
}
]
}
@@ -0,0 +1 @@
<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>transcribe<|end_message|><|message_user|><|content_audio_input|><|audio_end|><|end_message|><|message_user|><|content_audio_input|><|audio_end|><|end_message|><|message_model|>
@@ -0,0 +1,17 @@
{
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "look"
},
{
"type": "image_url",
"image_url": "data:image/png;base64,"
}
]
}
]
}
@@ -0,0 +1 @@
<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>look<|end_message|><|message_user|><|content_image|><|end_message|><|message_model|>
@@ -0,0 +1,46 @@
{
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather information",
"parameters": {
"type": "object",
"required": [
"city"
],
"properties": {
"city": {
"type": "string"
}
}
}
}
}
],
"messages": [
{
"role": "developer",
"content": "rules",
"tools": [
{
"type": "function",
"function": {
"name": "local_tool",
"parameters": {
"z": 1,
"a": {
"b": 2
}
}
}
}
]
},
{
"role": "user",
"content": "hi"
}
]
}
@@ -0,0 +1 @@
<|message_system|>tool_declare<|content_xml|>[{"description":"Get weather information","name":"get_weather","parameters":{"properties":{"city":{"type":"string"}},"required":["city"],"type":"object"},"type":"function"},{"description":"","name":"local_tool","parameters":{"a":{"b":2},"z":1},"type":"function"}]<|end_message|><|message_system|><|content_text|>rules<|end_message|><|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>hi<|end_message|><|message_model|>
@@ -0,0 +1,24 @@
{
"add_generation_prompt": false,
"messages": [
{
"role": "assistant",
"reasoning_content": "think",
"content": "answer",
"tool_calls": [
{
"id": "call_1",
"function": {
"name": "get_weather",
"arguments": "{\"city\":\"SF\"}"
}
}
]
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "sunny"
}
]
}
@@ -0,0 +1 @@
<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_model|><|content_thinking|>think<|end_message|><|message_model|><|content_text|>answer<|end_message|><|message_model|>get_weather<|content_invoke_tool_json|>{"name":"get_weather","args":{"city":"SF"}}<|end_message|><|content_model_end_sampling|><|message_tool|>get_weather<|content_text|>sunny<|end_message|>
+442
View File
@@ -0,0 +1,442 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use serde_json::{Map, Value, json};
use thiserror_ext::AsReport as _;
use vllm_text::Prompt;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use crate::error::{Error, Result};
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool};
use crate::{AssistantContentBlock, AssistantToolCall};
const MESSAGE_USER: &str = "<|message_user|>";
const MESSAGE_MODEL: &str = "<|message_model|>";
const MESSAGE_SYSTEM: &str = "<|message_system|>";
const MESSAGE_TOOL: &str = "<|message_tool|>";
const CONTENT_TEXT: &str = "<|content_text|>";
const CONTENT_IMAGE: &str = "<|content_image|>";
const CONTENT_MODEL_END_SAMPLING: &str = "<|content_model_end_sampling|>";
const CONTENT_AUDIO_INPUT: &str = "<|content_audio_input|>";
const CONTENT_THINKING: &str = "<|content_thinking|>";
// Inkling renderer semantics use this slot for structured payloads such as tool
// declarations.
const CONTENT_XML: &str = "<|content_xml|>";
const CONTENT_INVOKE_TOOL_JSON: &str = "<|content_invoke_tool_json|>";
const END_MESSAGE: &str = "<|end_message|>";
const AUDIO_END: &str = "<|audio_end|>";
const MAX_REASONING_EFFORT: f64 = 0.99;
/// Native Inkling renderer that emits token IDs directly.
#[derive(Clone)]
pub struct InklingChatRenderer {
tokenizer: DynTokenizer,
special: InklingSpecialTokenIds,
}
#[derive(Debug, Clone, Copy)]
struct InklingSpecialTokenIds {
message_user: u32,
message_model: u32,
message_system: u32,
message_tool: u32,
content_text: u32,
content_image: u32,
content_model_end_sampling: u32,
content_audio_input: u32,
content_thinking: u32,
content_xml: u32,
content_invoke_tool_json: u32,
end_message: u32,
audio_end: u32,
}
impl InklingSpecialTokenIds {
fn resolve(tokenizer: &dyn Tokenizer) -> Result<Self> {
Ok(Self {
message_user: resolve_special_token(tokenizer, MESSAGE_USER)?,
message_model: resolve_special_token(tokenizer, MESSAGE_MODEL)?,
message_system: resolve_special_token(tokenizer, MESSAGE_SYSTEM)?,
message_tool: resolve_special_token(tokenizer, MESSAGE_TOOL)?,
content_text: resolve_special_token(tokenizer, CONTENT_TEXT)?,
content_image: resolve_special_token(tokenizer, CONTENT_IMAGE)?,
content_model_end_sampling: resolve_special_token(
tokenizer,
CONTENT_MODEL_END_SAMPLING,
)?,
content_audio_input: resolve_special_token(tokenizer, CONTENT_AUDIO_INPUT)?,
content_thinking: resolve_special_token(tokenizer, CONTENT_THINKING)?,
content_xml: resolve_special_token(tokenizer, CONTENT_XML)?,
content_invoke_tool_json: resolve_special_token(tokenizer, CONTENT_INVOKE_TOOL_JSON)?,
end_message: resolve_special_token(tokenizer, END_MESSAGE)?,
audio_end: resolve_special_token(tokenizer, AUDIO_END)?,
})
}
}
impl InklingChatRenderer {
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
let special = InklingSpecialTokenIds::resolve(tokenizer.as_ref())?;
Ok(Self { tokenizer, special })
}
fn write_text_tokens(&self, out: &mut Vec<u32>, text: &str) -> Result<()> {
out.extend(self.tokenizer.encode(text, false)?);
Ok(())
}
fn write_message_start(
&self,
out: &mut Vec<u32>,
role_token_id: u32,
author_name: Option<&str>,
) -> Result<()> {
out.push(role_token_id);
if let Some(author_name) = author_name
&& !author_name.is_empty()
{
self.write_text_tokens(out, author_name)?;
}
Ok(())
}
fn write_text_block(
&self,
out: &mut Vec<u32>,
role_token_id: u32,
author_name: Option<&str>,
text: &str,
) -> Result<()> {
self.write_message_start(out, role_token_id, author_name)?;
out.push(self.special.content_text);
self.write_text_tokens(out, text)?;
out.push(self.special.end_message);
Ok(())
}
/// Write one image block holding only the `<|content_image|>` marker.
/// Multimodal preprocessing later expands the marker into per-patch
/// image placeholder tokens once the patch count is known, mirroring the
/// Python `InklingMultiModalProcessor` marker-anchored prompt updates.
fn write_image_block(&self, out: &mut Vec<u32>, role_token_id: u32) {
out.push(role_token_id);
out.push(self.special.content_image);
out.push(self.special.end_message);
}
/// Write one audio block holding the `<|content_audio_input|>` marker
/// followed by the `<|audio_end|>` terminator. Multimodal preprocessing
/// later expands the marker into per-frame audio placeholder tokens
/// (landing before `<|audio_end|>`) once the clip length is known.
fn write_audio_block(&self, out: &mut Vec<u32>, role_token_id: u32) {
out.push(role_token_id);
out.push(self.special.content_audio_input);
out.push(self.special.audio_end);
out.push(self.special.end_message);
}
fn write_reasoning_block(&self, out: &mut Vec<u32>, text: &str) -> Result<()> {
if text.is_empty() {
return Ok(());
}
out.push(self.special.message_model);
out.push(self.special.content_thinking);
self.write_text_tokens(out, text)?;
out.push(self.special.end_message);
Ok(())
}
fn write_reasoning_effort(&self, out: &mut Vec<u32>, effort: f64) -> Result<()> {
if !(0.0..=MAX_REASONING_EFFORT).contains(&effort) {
return Err(Error::ChatTemplate(format!(
"Inkling reasoning_effort must be in [0.0, 0.99], got {effort}"
)));
}
let formatted = format!("{effort:.2}");
let effort = formatted.trim_end_matches('0').trim_end_matches('.');
let effort = if matches!(effort, "0" | "-0") {
"0.0"
} else {
effort
};
self.write_text_block(
out,
self.special.message_system,
None,
&format!("Thinking effort level: {effort}"),
)
}
fn write_tool_declarations(&self, out: &mut Vec<u32>, tools: &[&ChatTool]) -> Result<()> {
if tools.is_empty() {
return Ok(());
}
let mut specs = Vec::with_capacity(tools.len());
for tool in tools {
specs.push(json!({
"description": tool.description.as_deref().unwrap_or(""),
"name": tool.name,
"parameters": sort_json(&tool.parameters),
"type": "function",
}));
}
let payload = compact_json(&sort_json(&Value::Array(specs)))?;
self.write_message_start(out, self.special.message_system, Some("tool_declare"))?;
out.push(self.special.content_xml);
self.write_text_tokens(out, &payload)?;
out.push(self.special.end_message);
Ok(())
}
fn write_chat_content(
&self,
out: &mut Vec<u32>,
role_token_id: u32,
content: &ChatContent,
) -> Result<()> {
match content {
ChatContent::Text(text) => {
if !text.is_empty() {
self.write_text_block(out, role_token_id, None, text)?;
}
}
ChatContent::Parts(parts) => {
for part in parts {
match part {
ChatContentPart::Text { text } => {
self.write_text_block(out, role_token_id, None, text)?;
}
ChatContentPart::ImageUrl { .. } => {
self.write_image_block(out, role_token_id);
}
ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => {
self.write_audio_block(out, role_token_id);
}
// Inkling has no video modality.
ChatContentPart::VideoUrl { .. } => {
return Err(Error::UnsupportedMultimodalContent("video_url"));
}
}
}
}
}
Ok(())
}
fn write_assistant_tool_call(
&self,
out: &mut Vec<u32>,
tool_call: &AssistantToolCall,
) -> Result<()> {
let payload = tool_call_json(tool_call)?;
self.write_message_start(out, self.special.message_model, Some(&tool_call.name))?;
out.push(self.special.content_invoke_tool_json);
self.write_text_tokens(out, &payload)?;
out.push(self.special.end_message);
Ok(())
}
fn write_assistant_content(
&self,
out: &mut Vec<u32>,
content: &[AssistantContentBlock],
tool_call_id_to_name: &mut HashMap<String, String>,
) -> Result<()> {
for block in content {
match block {
AssistantContentBlock::Reasoning { text } => {
self.write_reasoning_block(out, text)?;
}
AssistantContentBlock::Text { text } => {
if !text.is_empty() {
self.write_text_block(out, self.special.message_model, None, text)?;
}
}
AssistantContentBlock::ToolCall(tool_call) => {
if !tool_call.id.is_empty() {
tool_call_id_to_name.insert(tool_call.id.clone(), tool_call.name.clone());
}
self.write_assistant_tool_call(out, tool_call)?;
}
}
}
out.push(self.special.content_model_end_sampling);
Ok(())
}
fn write_tool_response(
&self,
out: &mut Vec<u32>,
content: &ChatContent,
tool_call_id: &str,
tool_call_id_to_name: &HashMap<String, String>,
) -> Result<()> {
let text = content.try_flatten_to_text()?;
let tool_name = tool_call_id_to_name.get(tool_call_id).map(String::as_str).unwrap_or("");
self.write_text_block(out, self.special.message_tool, Some(tool_name), &text)
}
}
impl ChatRenderer for InklingChatRenderer {
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
request.validate()?;
if request.chat_options.continue_final_message() {
return Err(Error::ChatTemplate(
"Inkling renderer does not support continue_final_message".to_string(),
));
}
let mut out = Vec::new();
let mut tool_call_id_to_name = HashMap::new();
let effective_template_kwargs = request_template_kwargs(request);
let tools = rendered_tools(request);
self.write_tool_declarations(&mut out, &tools)?;
let mut reasoning_effort =
resolve_reasoning_effort(effective_template_kwargs.get("reasoning_effort"));
for message in &request.messages {
if !matches!(
message,
ChatMessage::System { .. } | ChatMessage::Developer { .. }
) && let Some(effort) = reasoning_effort.take()
{
self.write_reasoning_effort(&mut out, effort)?;
}
match message {
ChatMessage::System { content } => {
self.write_chat_content(&mut out, self.special.message_system, content)?;
}
ChatMessage::Developer { content, .. } => {
self.write_chat_content(&mut out, self.special.message_system, content)?;
}
ChatMessage::User { content } => {
self.write_chat_content(&mut out, self.special.message_user, content)?;
}
ChatMessage::Assistant { content } => {
self.write_assistant_content(&mut out, content, &mut tool_call_id_to_name)?;
}
ChatMessage::ToolResponse {
content,
tool_call_id,
} => {
self.write_tool_response(
&mut out,
content,
tool_call_id,
&tool_call_id_to_name,
)?;
}
}
}
if let Some(effort) = reasoning_effort {
self.write_reasoning_effort(&mut out, effort)?;
}
if request.chat_options.add_generation_prompt() {
out.push(self.special.message_model);
}
Ok(RenderedPrompt {
prompt: Prompt::TokenIds(out),
effective_template_kwargs,
})
}
}
fn resolve_reasoning_effort(value: Option<&Value>) -> Option<f64> {
let Some(value) = value else {
return Some(0.9);
};
match value {
Value::String(name) => match name.as_str() {
"none" => Some(0.0),
"minimal" => Some(0.1),
"low" => Some(0.2),
"medium" => Some(0.7),
"high" => Some(0.9),
"xhigh" | "max" => Some(0.99),
_ => None,
},
Value::Number(number) => number.as_f64(),
_ => None,
}
}
fn resolve_special_token(tokenizer: &dyn Tokenizer, token: &str) -> Result<u32> {
tokenizer.token_to_id(token).ok_or_else(|| {
Error::ChatTemplate(format!(
"Inkling tokenizer is missing special token `{token}`"
))
})
}
fn rendered_tools(request: &ChatRequest) -> Vec<&ChatTool> {
if !request.tool_parsing_enabled() {
return Vec::new();
}
let mut tools = Vec::with_capacity(request.tools.len());
tools.extend(request.tools.iter());
for message in &request.messages {
if let ChatMessage::Developer {
tools: Some(local_tools),
..
} = message
{
tools.extend(local_tools.iter());
}
}
tools
}
fn tool_call_json(tool_call: &AssistantToolCall) -> Result<String> {
let name_json = serde_json::to_string(&tool_call.name)
.map_err(|error| Error::ChatTemplate(error.as_report().to_string()))?;
let arguments = if tool_call.arguments.trim().is_empty() {
Value::Object(Map::new())
} else {
serde_json::from_str(&tool_call.arguments).map_err(|error| {
Error::ChatTemplate(format!(
"Inkling tool call arguments must decode to a JSON object: {error}"
))
})?
};
let Value::Object(_) = arguments else {
return Err(Error::ChatTemplate(
"Inkling tool call arguments must decode to a JSON object".to_string(),
));
};
let args_json = compact_json(&sort_json(&arguments))?;
Ok(format!("{{\"name\":{name_json},\"args\":{args_json}}}"))
}
fn compact_json(value: &Value) -> Result<String> {
serde_json::to_string(value).map_err(|error| Error::ChatTemplate(error.as_report().to_string()))
}
fn sort_json(value: &Value) -> Value {
match value {
Value::Array(items) => Value::Array(items.iter().map(sort_json).collect()),
Value::Object(map) => {
let mut sorted = Map::new();
let mut keys = map.keys().collect::<Vec<_>>();
keys.sort();
for key in keys {
sorted.insert(key.clone(), sort_json(&map[key]));
}
Value::Object(sorted)
}
_ => value.clone(),
}
}
#[cfg(test)]
mod tests;
+375
View File
@@ -0,0 +1,375 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::path::PathBuf;
use std::sync::Arc;
use expect_test::{ExpectFile, expect_file};
use serde_json::json;
use thiserror_ext::AsReport;
use vllm_text::tokenizer::Tokenizer;
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
use super::{
AUDIO_END, CONTENT_AUDIO_INPUT, CONTENT_IMAGE, CONTENT_INVOKE_TOOL_JSON, CONTENT_TEXT,
CONTENT_THINKING, CONTENT_XML, END_MESSAGE, InklingChatRenderer, MESSAGE_MODEL, MESSAGE_SYSTEM,
MESSAGE_TOOL, MESSAGE_USER,
};
use crate::event::{AssistantContentBlock, AssistantToolCall};
use crate::request::{ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort};
use crate::{ChatRenderer, Error};
struct FixtureTokenizer;
impl Tokenizer for FixtureTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
Ok(text.bytes().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<|message_user|>" => Some(200000),
"<|message_model|>" => Some(200001),
"<|message_system|>" => Some(200002),
"<|message_tool|>" => Some(200003),
"<|content_text|>" => Some(200004),
"<|content_image|>" => Some(200005),
"<|content_model_end_sampling|>" => Some(200006),
"<|content_thinking|>" => Some(200008),
"<|end_message|>" => Some(200010),
"<|content_audio_input|>" => Some(200020),
CONTENT_XML => Some(200024),
"<|audio_end|>" => Some(200043),
"<|content_invoke_tool_json|>" => Some(200049),
_ => None,
}
}
fn id_to_token(&self, id: u32) -> Option<String> {
let token = match id {
200000 => "<|message_user|>",
200001 => "<|message_model|>",
200002 => "<|message_system|>",
200003 => "<|message_tool|>",
200004 => "<|content_text|>",
200005 => "<|content_image|>",
200006 => "<|content_model_end_sampling|>",
200008 => "<|content_thinking|>",
200010 => "<|end_message|>",
200020 => "<|content_audio_input|>",
200024 => CONTENT_XML,
200043 => "<|audio_end|>",
200049 => "<|content_invoke_tool_json|>",
_ => return None,
};
Some(token.to_string())
}
}
fn renderer() -> InklingChatRenderer {
InklingChatRenderer::new(Arc::new(FixtureTokenizer)).unwrap()
}
fn render_token_ids(request: &ChatRequest) -> Vec<u32> {
renderer()
.render(request)
.unwrap()
.prompt
.into_token_ids()
.expect("Inkling renderer returns token IDs")
}
fn fixture_request(name: &str) -> ChatRequest {
fixture_chat_request(&fixture_path(name), inkling_fixture_options())
}
fn inkling_fixture_options() -> FixtureRequestOptions {
FixtureRequestOptions {
enable_thinking: false,
no_generation_prompt_when_last_assistant: false,
}
}
fn fixture_path(name: &str) -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("src/renderer/inkling")
.join("fixtures")
.join(name)
}
fn assert_fixture(input_name: &str, expected: ExpectFile) {
let request = fixture_request(input_name);
let rendered = format!("{}\n", render_symbolic_tokens(&render_token_ids(&request)));
expected.assert_eq(&rendered);
}
fn render_symbolic_tokens(token_ids: &[u32]) -> String {
let mut out = String::new();
let mut text_bytes = Vec::new();
for token_id in token_ids {
if let Some(marker) = symbolic_tokens()
.iter()
.find_map(|(marker, marker_id)| (token_id == marker_id).then_some(*marker))
{
flush_text_bytes(&mut out, &mut text_bytes);
out.push_str(marker);
continue;
}
let byte = u8::try_from(*token_id)
.unwrap_or_else(|_| panic!("unexpected non-byte fixture text token {token_id}"));
text_bytes.push(byte);
}
flush_text_bytes(&mut out, &mut text_bytes);
out
}
fn flush_text_bytes(out: &mut String, text_bytes: &mut Vec<u8>) {
if text_bytes.is_empty() {
return;
}
out.push_str(std::str::from_utf8(text_bytes).unwrap());
text_bytes.clear();
}
fn symbolic_tokens() -> [(&'static str, u32); 13] {
[
(MESSAGE_USER, 200000),
(MESSAGE_MODEL, 200001),
(MESSAGE_SYSTEM, 200002),
(MESSAGE_TOOL, 200003),
(CONTENT_TEXT, 200004),
(CONTENT_IMAGE, 200005),
(super::CONTENT_MODEL_END_SAMPLING, 200006),
(CONTENT_THINKING, 200008),
(END_MESSAGE, 200010),
(CONTENT_AUDIO_INPUT, 200020),
(CONTENT_XML, 200024),
(AUDIO_END, 200043),
(CONTENT_INVOKE_TOOL_JSON, 200049),
]
}
#[test]
fn renders_text_and_image_as_inkling_tokens() {
assert_fixture(
"text_image_input.json",
expect_file!["fixtures/text_image_output.txt"],
);
}
#[test]
fn renders_audio_marker_only_blocks() {
assert_fixture(
"text_audio_input.json",
expect_file!["fixtures/text_audio_output.txt"],
);
}
#[test]
fn renders_reasoning_text_tool_call_and_tool_response() {
assert_fixture(
"tool_round_trip_input.json",
expect_file!["fixtures/tool_round_trip_output.txt"],
);
}
#[test]
fn renders_request_and_developer_tools_as_tool_declare() {
assert_fixture(
"tool_declare_input.json",
expect_file!["fixtures/tool_declare_output.txt"],
);
}
#[test]
fn renders_named_reasoning_effort_after_tool_declarations() {
for (effort, expected) in [
(ReasoningEffort::None, "0.0"),
(ReasoningEffort::Minimal, "0.1"),
(ReasoningEffort::Low, "0.2"),
(ReasoningEffort::Medium, "0.7"),
(ReasoningEffort::High, "0.9"),
(ReasoningEffort::XHigh, "0.99"),
(ReasoningEffort::Max, "0.99"),
] {
let mut request = fixture_request("tool_declare_input.json");
request.chat_options.reasoning_effort = Some(effort);
let rendered = render_symbolic_tokens(&render_token_ids(&request));
let tool_end = rendered.find("<|end_message|>").unwrap();
let effort_block = format!(
"<|message_system|><|content_text|>Thinking effort level: \
{expected}<|end_message|>"
);
let effort_start = rendered.find(&effort_block).unwrap();
let system_end = rendered.find("rules<|end_message|>").unwrap();
assert!(effort_start > tool_end);
assert!(effort_start > system_end);
}
}
#[test]
fn emits_one_reasoning_effort_for_multi_turn_conversation() {
let request = ChatRequest {
messages: vec![
ChatMessage::system("rules"),
ChatMessage::user("user1"),
ChatMessage::assistant_text("assistant1"),
ChatMessage::user("user2"),
],
..ChatRequest::for_test()
};
let rendered = render_symbolic_tokens(&render_token_ids(&request));
let effort = "<|message_system|><|content_text|>Thinking effort level: 0.9\
<|end_message|>";
assert_eq!(rendered.matches(effort).count(), 1);
assert!(rendered.find("rules").unwrap() < rendered.find(effort).unwrap());
assert!(rendered.find(effort).unwrap() < rendered.find("user1").unwrap());
assert!(rendered.find("user1").unwrap() < rendered.find("assistant1").unwrap());
assert!(rendered.find("assistant1").unwrap() < rendered.find("user2").unwrap());
}
#[test]
fn canonicalizes_numeric_zero_reasoning_effort() {
for value in [0.0, -0.0] {
let mut request = ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("reasoning_effort".to_string(), json!(value));
assert!(
render_symbolic_tokens(&render_token_ids(&request)).starts_with(
"<|message_system|><|content_text|>Thinking effort level: 0.0\
<|end_message|>"
)
);
}
}
#[test]
fn defaults_reasoning_effort_to_high() {
let request = ChatRequest::for_test();
assert!(
render_symbolic_tokens(&render_token_ids(&request)).starts_with(
"<|message_system|><|content_text|>Thinking effort level: 0.9\
<|end_message|>"
)
);
}
#[test]
fn renders_numeric_reasoning_effort_template_kwarg() {
let mut request = ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("reasoning_effort".to_string(), json!(0.8));
assert_eq!(
render_symbolic_tokens(&render_token_ids(&request)),
"<|message_system|><|content_text|>Thinking effort level: 0.8\
<|end_message|><|message_user|><|content_text|>test\
<|end_message|><|message_model|>"
);
}
#[test]
fn ignores_unsupported_reasoning_effort_values() {
for value in [json!(true), json!("invalid"), json!(null)] {
let mut request = ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("reasoning_effort".to_string(), value);
assert_eq!(
render_symbolic_tokens(&render_token_ids(&request)),
"<|message_user|><|content_text|>test<|end_message|><|message_model|>"
);
}
}
#[test]
fn rejects_out_of_range_reasoning_effort() {
for value in [0.990_000_1, 1.0, 1.5, -0.1] {
let mut request = ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("reasoning_effort".to_string(), json!(value));
let error = renderer().render(&request).unwrap_err();
assert!(error.as_report().to_string().contains("must be in [0.0, 0.99]"));
}
}
#[test]
fn rejects_continue_final_message() {
let request = ChatRequest {
messages: vec![ChatMessage::assistant_text("partial")],
chat_options: crate::ChatOptions {
generation_prompt_mode: GenerationPromptMode::ContinueFinalAssistant,
..Default::default()
},
..ChatRequest::for_test()
};
let error = renderer().render(&request).unwrap_err();
assert!(matches!(error, Error::ChatTemplate(_)));
assert!(error.as_report().to_string().contains("continue_final_message"));
}
#[test]
fn renders_developer_messages_as_system() {
let request = ChatRequest {
messages: vec![ChatMessage::developer("rules", None)],
..ChatRequest::for_test()
};
assert_eq!(
render_symbolic_tokens(&render_token_ids(&request)),
"<|message_system|><|content_text|>rules<|end_message|>\
<|message_system|><|content_text|>Thinking effort level: 0.9\
<|end_message|><|message_model|>"
);
}
#[test]
fn rejects_non_object_tool_call_arguments() {
let request = ChatRequest {
messages: vec![ChatMessage::assistant_blocks(vec![
AssistantContentBlock::ToolCall(AssistantToolCall {
id: "call_1".to_string(),
name: "get_weather".to_string(),
arguments: "[]".to_string(),
}),
])],
..ChatRequest::for_test()
};
let error = renderer().render(&request).unwrap_err();
assert!(error.as_report().to_string().contains("JSON object"));
}
+2
View File
@@ -14,6 +14,7 @@ pub mod deepseek_v32;
pub mod deepseek_v4;
pub mod harmony;
pub mod hf;
mod inkling;
mod selection;
#[cfg(test)]
mod test_utils;
@@ -21,6 +22,7 @@ mod test_utils;
pub use deepseek_v4::DeepSeekV4ChatRenderer;
pub use deepseek_v32::DeepSeekV32ChatRenderer;
pub use harmony::HarmonyChatRenderer;
pub use inkling::InklingChatRenderer;
pub use selection::RendererSelection;
/// Rendered chat prompt submitted to the text backend.
+9 -1
View File
@@ -24,6 +24,8 @@ pub enum RendererSelection {
DeepSeekV4,
/// Force the GPT-OSS Harmony renderer.
Harmony,
/// Force the Inkling native token renderer.
Inkling,
}
impl RendererSelection {
@@ -33,6 +35,8 @@ impl RendererSelection {
pub const GPT_OSS_MODEL_TYPE: &str = "gpt_oss";
pub const HARMONY_LITERAL: &str = "harmony";
pub const HF_LITERAL: &str = "hf";
pub const INKLING_LITERAL: &str = "inkling";
pub const INKLING_MODEL_TYPE: &str = "inkling_mm_model";
/// Resolve the renderer selection using the given model type string, if
/// it's `Auto`.
@@ -42,6 +46,7 @@ impl RendererSelection {
Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32,
Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
Self::GPT_OSS_MODEL_TYPE => Self::Harmony,
Self::INKLING_MODEL_TYPE => Self::Inkling,
_ => Self::Hf,
},
selection => selection,
@@ -63,6 +68,8 @@ impl FromStr for RendererSelection {
Ok(Self::DeepSeekV4)
} else if value.eq_ignore_ascii_case(Self::HARMONY_LITERAL) {
Ok(Self::Harmony)
} else if value.eq_ignore_ascii_case(Self::INKLING_LITERAL) {
Ok(Self::Inkling)
} else {
Err(format!(
"unknown renderer `{value}` (expected one of: {})",
@@ -80,6 +87,7 @@ impl fmt::Display for RendererSelection {
Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL),
Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
Self::Harmony => f.write_str(Self::HARMONY_LITERAL),
Self::Inkling => f.write_str(Self::INKLING_LITERAL),
}
}
}
@@ -106,7 +114,7 @@ mod tests {
fn renderer_selection_expected_error_message() {
let err = RendererSelection::from_str("unknown").unwrap_err();
expect_test::expect![
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)"
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling)"
]
.assert_eq(&err);
}
+11
View File
@@ -103,6 +103,8 @@ pub(crate) enum FixtureContent {
pub(crate) enum FixtureContentPart {
Text { text: String },
ImageUrl { image_url: String },
InputAudio { data: String },
AudioUrl { audio_url: String },
}
#[derive(Debug, Deserialize)]
@@ -226,6 +228,15 @@ fn to_chat_content(content: FixtureContent) -> ChatContent {
FixtureContentPart::ImageUrl { image_url } => {
ChatContentPart::image_url(image_url)
}
FixtureContentPart::InputAudio { data } => ChatContentPart::InputAudio {
data,
format: None,
uuid: None,
},
FixtureContentPart::AudioUrl { audio_url } => ChatContentPart::AudioUrl {
audio_url,
uuid: None,
},
})
.collect(),
),
+16 -2
View File
@@ -263,19 +263,32 @@ impl RoundtripCase {
sort_json_keys: false,
}
}
/// Inkling typed content blocks with native token-id rendering.
fn inkling() -> Self {
Self {
model_id: "thinkingmachines/Inkling",
assistant_stop_suffix: "",
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
thinking_behavior: ThinkingBehavior::Always { value: true },
json_fmt: compact_json_fmt(),
sort_json_keys: true,
}
}
}
macro_rules! roundtrip_tests {
($($case:ident => [$($(#[$fixture_attr:meta])* $fixture:ident),* $(,)?]),+ $(,)?) => {
($($case:ident => $(#[$case_attr:meta])* [$($fixture:ident),* $(,)?]),+ $(,)?) => {
paste::paste! {
$(
#[tokio::test]
#[file_serial([<hf_ $case>])]
$(#[$case_attr])*
async fn [<roundtrip_ $case>]() -> Result<()> {
let case = RoundtripCase::$case();
let backends = load_roundtrip_backends(&case).await?;
$(
$(#[$fixture_attr])*
[<run_roundtrip_ $fixture>](&case, &backends).await?;
)*
Ok(())
@@ -300,6 +313,7 @@ roundtrip_tests! {
gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call
kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history
gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call
inkling => [reasoning_and_content, tool_call_mix],
}
/// Run the fixed reasoning+content fixture for one model/parser case.
+1 -1
View File
@@ -646,7 +646,7 @@ fn serve_args_reject_unknown_renderer_value() {
.unwrap_err();
expect![[r#"
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling)
For more information, try '--help'.
"#]]
+12 -12
View File
@@ -31,24 +31,24 @@ use super::utils::{
};
use super::{Result, ToolCallDelta, ToolParserOutput};
type JsonToolInput<'i> = Partial<&'i str>;
pub(crate) type JsonToolInput<'i> = Partial<&'i str>;
#[derive(Debug, Clone, Copy)]
struct JsonToolCallConfig {
parser_name: &'static str,
start_marker: &'static str,
end_marker: &'static str,
marker_whitespace: JsonToolCallWhitespace,
delimiter: Option<&'static str>,
name_key: &'static str,
pub(crate) struct JsonToolCallConfig {
pub parser_name: &'static str,
pub start_marker: &'static str,
pub end_marker: &'static str,
pub marker_whitespace: JsonToolCallWhitespace,
pub delimiter: Option<&'static str>,
pub name_key: &'static str,
/// Candidate JSON keys naming the arguments payload, tried in order.
/// Most parsers use a single key like `["arguments"]`, but some accept
/// multiple (e.g. InternLM2 accepts `parameters` or `arguments`).
arguments_key: &'static [&'static str],
pub arguments_key: &'static [&'static str],
}
#[derive(Debug, Clone, Copy)]
enum JsonToolCallWhitespace {
pub(crate) enum JsonToolCallWhitespace {
Optional,
Exact(&'static str),
}
@@ -61,7 +61,7 @@ enum JsonToolCallMode {
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum JsonToolCallEvent {
pub(crate) enum JsonToolCallEvent {
Text { len: usize },
ToolCallStart,
ToolCallHeader { function_name: String },
@@ -220,7 +220,7 @@ fn tool_call_start_event(
/// Parse a marker-wrapped JSON tool-call header before the raw arguments
/// payload.
fn tool_call_header_event(
pub(crate) fn tool_call_header_event(
input: &mut JsonToolInput<'_>,
config: JsonToolCallConfig,
) -> ModalResult<JsonToolCallEvent> {
+1 -1
View File
@@ -9,7 +9,7 @@ mod deepseek_dsml;
pub(crate) mod deepseek_json;
mod glm_xml;
mod hy_v3;
mod json;
pub(crate) mod json;
mod kimi_k2;
mod minimax_m2;
mod minimax_m3;
+5 -14
View File
@@ -10,7 +10,7 @@ use winnow::prelude::*;
use winnow::stream::{Partial, Stream};
use winnow::token::{literal, take_till, take_until};
use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
use super::{Result, UnifiedParser, UnifiedParserOutput, token_id};
use crate::reasoning::last_reasoning_boundary;
use crate::tool::{Tool, ToolCallDelta};
use crate::unified::parsing_failed;
@@ -79,17 +79,8 @@ pub struct Gemma4UnifiedParser {
impl Gemma4UnifiedParser {
/// Create a Gemma4 parser.
pub fn new(_tools: &[Tool], tokenizer: DynTokenizer) -> Result<Self> {
let channel_start_token_id = tokenizer.token_to_id(CHANNEL_START).ok_or_else(|| {
UnifiedParserError::MissingToken {
token: CHANNEL_START.to_string(),
}
})?;
let channel_end_token_id =
tokenizer
.token_to_id(CHANNEL_END)
.ok_or_else(|| UnifiedParserError::MissingToken {
token: CHANNEL_END.to_string(),
})?;
let channel_start_token_id = token_id(tokenizer.as_ref(), CHANNEL_START)?;
let channel_end_token_id = token_id(tokenizer.as_ref(), CHANNEL_END)?;
Ok(Self {
buffer: String::new(),
@@ -524,10 +515,10 @@ mod tests {
use super::{
CHANNEL_END, CHANNEL_START, Gemma4UnifiedParser, ToolCallDelta, UnifiedParser,
UnifiedParserError, UnifiedParserOutput, gemma4_array_content, parse_gemma4_args,
UnifiedParserOutput, gemma4_array_content, parse_gemma4_args,
};
use crate::tool::Tool;
use crate::unified::{UnifiedParserEvent, parsing_failed};
use crate::unified::{UnifiedParserError, UnifiedParserEvent, parsing_failed};
const CHANNEL_START_ID: u32 = 256;
const CHANNEL_END_ID: u32 = 257;
+768
View File
@@ -0,0 +1,768 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use winnow::ascii::multispace0 as ws0;
use winnow::combinator::{alt, seq};
use winnow::error::ModalResult;
use winnow::prelude::*;
use winnow::stream::Partial;
use winnow::token::literal;
use vllm_tokenizer::DynTokenizer;
use super::{Result, UnifiedParser, UnifiedParserOutput, token_id};
use crate::tool::json::{
JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput,
tool_call_header_event,
};
use crate::tool::{Tool, ToolCallDelta};
use crate::unified::parsing_failed;
use crate::utils::{
JsonObjectScanState, parse_buffered_event, safe_text_len_mul, take_json_object,
};
const CONTENT_TEXT: &str = "<|content_text|>";
const CONTENT_THINKING: &str = "<|content_thinking|>";
const CONTENT_INVOKE_TOOL_JSON: &str = "<|content_invoke_tool_json|>";
const CONTENT_INVOKE_TOOL_TEXT: &str = "<|content_invoke_tool_text|>";
const CONTENT_MODEL_END_SAMPLING: &str = "<|content_model_end_sampling|>";
const CONTENT_TOOL_ERROR: &str = "<|content_tool_error|>";
const MESSAGE_MODEL: &str = "<|message_model|>";
const END_MESSAGE: &str = "<|end_message|>";
const IDLE_MARKERS: &[&str] = &[
MESSAGE_MODEL,
CONTENT_TEXT,
CONTENT_THINKING,
CONTENT_INVOKE_TOOL_JSON,
CONTENT_INVOKE_TOOL_TEXT,
CONTENT_MODEL_END_SAMPLING,
CONTENT_TOOL_ERROR,
END_MESSAGE,
];
const BLOCK_END_MARKERS: &[&str] = &[END_MESSAGE, CONTENT_MODEL_END_SAMPLING];
const INKLING_TOOL_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
parser_name: "Inkling",
start_marker: CONTENT_INVOKE_TOOL_JSON,
end_marker: END_MESSAGE,
marker_whitespace: JsonToolCallWhitespace::Optional,
delimiter: None,
name_key: "name",
arguments_key: &["args"],
};
type InklingInput<'i> = Partial<&'i str>;
#[derive(Debug, Clone, PartialEq, Eq)]
enum InklingEvent {
Text { len: usize },
Reasoning { len: usize },
TextStart,
ReasoningStart,
MessageStart,
Header,
ToolJsonStart,
ToolJsonHeader { name: String },
ToolJsonArgs { len: usize, complete: bool },
BlockEnd,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
enum InklingMode {
#[default]
Idle,
MessageHeader,
Text,
Reasoning,
ToolJsonHeader,
ToolJsonArgs {
json_scan: JsonObjectScanState,
},
ToolJsonClose,
}
/// Unified parser for Inkling typed content blocks.
pub struct InklingUnifiedParser {
buffer: String,
mode: InklingMode,
emitted_tool_count: usize,
active_tool_index: Option<usize>,
tokenizer: DynTokenizer,
message_model_token_id: u32,
content_text_token_id: u32,
content_thinking_token_id: u32,
}
impl InklingUnifiedParser {
/// Create a Inkling parser.
pub fn new(_tools: &[Tool], tokenizer: DynTokenizer) -> Result<Self> {
let message_model_token_id = token_id(tokenizer.as_ref(), MESSAGE_MODEL)?;
let content_text_token_id = token_id(tokenizer.as_ref(), CONTENT_TEXT)?;
let content_thinking_token_id = token_id(tokenizer.as_ref(), CONTENT_THINKING)?;
Ok(Self {
buffer: String::new(),
mode: InklingMode::Idle,
emitted_tool_count: 0,
active_tool_index: None,
tokenizer,
message_model_token_id,
content_text_token_id,
content_thinking_token_id,
})
}
fn initialize_mode(&mut self, prompt_token_ids: &[u32]) {
self.mode = InklingMode::Idle;
for token_id in prompt_token_ids.iter().rev().copied() {
if token_id == self.message_model_token_id {
self.mode = InklingMode::MessageHeader;
return;
}
if token_id == self.content_thinking_token_id {
self.mode = InklingMode::Reasoning;
return;
}
if token_id == self.content_text_token_id {
self.mode = InklingMode::Text;
return;
}
if self.tokenizer.is_special_id(token_id) {
return;
}
}
}
fn apply_event(&mut self, event: InklingEvent, output: &mut UnifiedParserOutput) -> Result<()> {
match event {
InklingEvent::Text { len } => output.push_text(self.buffer[..len].to_string()),
InklingEvent::Reasoning { len } => {
output.push_reasoning(self.buffer[..len].to_string());
}
InklingEvent::MessageStart => self.mode = InklingMode::MessageHeader,
InklingEvent::Header => {}
InklingEvent::TextStart => self.mode = InklingMode::Text,
InklingEvent::ReasoningStart => self.mode = InklingMode::Reasoning,
InklingEvent::ToolJsonStart => self.mode = InklingMode::ToolJsonHeader,
InklingEvent::ToolJsonHeader { name } => {
let tool_index = self.emitted_tool_count;
self.emitted_tool_count += 1;
self.active_tool_index = Some(tool_index);
self.mode = InklingMode::ToolJsonArgs {
json_scan: JsonObjectScanState::default(),
};
output.push_call(ToolCallDelta {
tool_index,
name: Some(name),
arguments: String::new(),
});
}
InklingEvent::ToolJsonArgs { len, complete } => {
let Some(tool_index) = self.active_tool_index else {
return Err(parsing_failed!(
"Inkling arguments without an active tool call"
));
};
output.push_call(ToolCallDelta {
tool_index,
name: None,
arguments: self.buffer[..len].to_string(),
});
if complete {
self.mode = InklingMode::ToolJsonClose;
}
}
InklingEvent::BlockEnd => {
self.mode = InklingMode::Idle;
self.active_tool_index = None;
}
}
Ok(())
}
fn reset(&mut self) -> String {
self.mode = InklingMode::Idle;
self.active_tool_index = None;
self.emitted_tool_count = 0;
std::mem::take(&mut self.buffer)
}
}
impl UnifiedParser for InklingUnifiedParser {
fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
where
Self: Sized + 'static,
{
Self::new(tools, tokenizer).map(|parser| Box::new(parser) as Box<dyn UnifiedParser>)
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.buffer.clear();
self.emitted_tool_count = 0;
self.active_tool_index = None;
self.initialize_mode(prompt_token_ids);
Ok(())
}
fn preserve_special_tokens(&self) -> bool {
true
}
fn parse_into(&mut self, chunk: &str, output: &mut UnifiedParserOutput) -> Result<()> {
self.buffer.push_str(chunk);
while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| {
parse_next_inkling_event(input, &mut self.mode)
})? {
self.apply_event(event, output)?;
self.buffer.drain(..consumed_len);
}
Ok(())
}
fn finish(&mut self) -> Result<UnifiedParserOutput> {
let mut output = UnifiedParserOutput::default();
match &self.mode {
InklingMode::Idle | InklingMode::Text => {
output.push_text(std::mem::take(&mut self.buffer))
}
InklingMode::MessageHeader => self.buffer.clear(),
InklingMode::Reasoning => output.push_reasoning(std::mem::take(&mut self.buffer)),
InklingMode::ToolJsonHeader
| InklingMode::ToolJsonArgs { .. }
| InklingMode::ToolJsonClose => {
return Err(parsing_failed!("incomplete Inkling tool call"));
}
}
let _ = self.reset();
Ok(output)
}
fn reset(&mut self) -> String {
InklingUnifiedParser::reset(self)
}
}
/// Parse one Inkling event from buffered streaming input.
fn parse_next_inkling_event(
input: &mut InklingInput<'_>,
mode: &mut InklingMode,
) -> ModalResult<InklingEvent> {
match mode {
InklingMode::Idle => parse_idle_event(input),
InklingMode::MessageHeader => parse_message_header_event(input),
InklingMode::Text => parse_text_event(input),
InklingMode::Reasoning => parse_reasoning_event(input),
InklingMode::ToolJsonHeader => parse_tool_json_header_event(input),
InklingMode::ToolJsonArgs { json_scan } => parse_tool_json_args_event(input, json_scan),
InklingMode::ToolJsonClose => parse_tool_json_close_event(input),
}
}
/// Parse an event while waiting for a Inkling content kind.
fn parse_idle_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
alt((
message_start_event,
reasoning_start_event,
text_start_event,
tool_json_start_event,
raw_text_start_event,
block_end_event,
safe_idle_text_event,
))
.parse_next(input)
}
/// Parse a Inkling model-authored message start marker.
fn message_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
literal(MESSAGE_MODEL).value(InklingEvent::MessageStart).parse_next(input)
}
/// Parse an event while waiting for an Inkling message content kind.
fn parse_message_header_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
alt((
reasoning_start_event,
text_start_event,
tool_json_start_event,
raw_text_start_event,
block_end_event,
safe_header_event,
))
.parse_next(input)
}
/// Parse an event inside a Inkling text block.
fn parse_text_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
alt((block_end_event, safe_text_event)).parse_next(input)
}
/// Parse an event inside a Inkling reasoning block.
fn parse_reasoning_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
alt((block_end_event, safe_reasoning_event)).parse_next(input)
}
/// Parse a Inkling text start marker.
fn text_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
literal(CONTENT_TEXT).value(InklingEvent::TextStart).parse_next(input)
}
/// Parse a Inkling reasoning start marker.
fn reasoning_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
literal(CONTENT_THINKING).value(InklingEvent::ReasoningStart).parse_next(input)
}
/// Parse a Inkling JSON tool-call start marker.
fn tool_json_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
literal(CONTENT_INVOKE_TOOL_JSON)
.value(InklingEvent::ToolJsonStart)
.parse_next(input)
}
/// Parse a Inkling content kind treated as visible text.
fn raw_text_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
alt((
literal(CONTENT_INVOKE_TOOL_TEXT),
literal(CONTENT_TOOL_ERROR),
))
.value(InklingEvent::TextStart)
.parse_next(input)
}
/// Parse a Inkling block end marker.
fn block_end_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
alt((literal(END_MESSAGE), literal(CONTENT_MODEL_END_SAMPLING)))
.value(InklingEvent::BlockEnd)
.parse_next(input)
}
/// Parse safe text while waiting for the next Inkling marker.
fn safe_idle_text_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
safe_text_len_mul(input, IDLE_MARKERS).map(|len| InklingEvent::Text { len })
}
/// Parse safe header text before the next Inkling marker.
fn safe_header_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
safe_text_len_mul(input, IDLE_MARKERS).map(|_| InklingEvent::Header)
}
/// Parse safe text before the end of a Inkling text block.
fn safe_text_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
safe_text_len_mul(input, BLOCK_END_MARKERS).map(|len| InklingEvent::Text { len })
}
/// Parse safe reasoning before the end of a Inkling reasoning block.
fn safe_reasoning_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
safe_text_len_mul(input, BLOCK_END_MARKERS).map(|len| InklingEvent::Reasoning { len })
}
/// Parse a Inkling JSON tool-call header.
fn parse_tool_json_header_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
match tool_call_header_event(input, INKLING_TOOL_CONFIG)? {
JsonToolCallEvent::ToolCallHeader { function_name } => Ok(InklingEvent::ToolJsonHeader {
name: function_name,
}),
_ => unreachable!("tool_call_header_event only emits ToolCallHeader"),
}
}
/// Parse raw Inkling JSON tool-call argument bytes.
fn parse_tool_json_args_event(
input: &mut JsonToolInput<'_>,
json_scan: &mut JsonObjectScanState,
) -> ModalResult<InklingEvent> {
let len = take_json_object(input, json_scan)?;
Ok(InklingEvent::ToolJsonArgs {
len,
complete: json_scan.complete(),
})
}
/// Parse the close of a Inkling JSON tool-call block.
fn parse_tool_json_close_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
seq!(
_: ws0,
_: literal("}"),
_: ws0,
_: literal(END_MESSAGE),
)
.value(InklingEvent::BlockEnd)
.parse_next(input)
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use super::{CONTENT_TEXT, CONTENT_THINKING, InklingUnifiedParser, MESSAGE_MODEL};
use crate::tool::Tool;
use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
use thiserror_ext::AsReport;
use vllm_tokenizer::Tokenizer;
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
MESSAGE_MODEL => Some(200001),
"<|content_text|>" => Some(200004),
"<|content_model_end_sampling|>" => Some(200006),
"<|content_thinking|>" => Some(200008),
"<|end_message|>" => Some(200010),
"<|content_tool_error|>" => Some(200022),
"<|content_invoke_tool_json|>" => Some(200049),
"<|content_invoke_tool_text|>" => Some(200057),
_ => None,
}
}
fn id_to_token(&self, id: u32) -> Option<String> {
let token = match id {
200001 => MESSAGE_MODEL,
200004 => "<|content_text|>",
200006 => "<|content_model_end_sampling|>",
200008 => "<|content_thinking|>",
200010 => "<|end_message|>",
200022 => "<|content_tool_error|>",
200049 => "<|content_invoke_tool_json|>",
200057 => "<|content_invoke_tool_text|>",
_ => return None,
};
Some(token.to_string())
}
fn is_special_id(&self, token_id: u32) -> bool {
(199999..=200057).contains(&token_id)
}
}
trait UnifiedParserTestExt {
fn parse_chunk(&mut self, chunk: &str) -> super::Result<UnifiedParserOutput>;
fn parse_complete(&mut self, text: &str) -> super::Result<UnifiedParserOutput>;
}
impl<T: UnifiedParser + ?Sized> UnifiedParserTestExt for T {
fn parse_chunk(&mut self, chunk: &str) -> super::Result<UnifiedParserOutput> {
let mut output = UnifiedParserOutput::default();
self.parse_into(chunk, &mut output)?;
Ok(output)
}
fn parse_complete(&mut self, text: &str) -> super::Result<UnifiedParserOutput> {
let mut output = self.parse_chunk(text)?;
output.append(self.finish()?);
Ok(output)
}
}
trait UnifiedParserOutputTestExt {
fn normal_text(&self) -> String;
fn reasoning_text(&self) -> String;
fn calls(&self) -> Vec<crate::tool::ToolCallDelta>;
}
impl UnifiedParserOutputTestExt for UnifiedParserOutput {
fn normal_text(&self) -> String {
self.events
.iter()
.filter_map(|event| match event {
UnifiedParserEvent::Text(text) => Some(text.as_str()),
_ => None,
})
.collect()
}
fn reasoning_text(&self) -> String {
self.events
.iter()
.filter_map(|event| match event {
UnifiedParserEvent::Reasoning(text) => Some(text.as_str()),
_ => None,
})
.collect()
}
fn calls(&self) -> Vec<crate::tool::ToolCallDelta> {
self.events
.iter()
.filter_map(|event| match event {
UnifiedParserEvent::ToolCall(call) => Some(call.clone()),
_ => None,
})
.collect()
}
}
fn test_tools() -> Vec<Tool> {
vec![Tool {
name: "get_weather".to_string(),
description: Some("Get weather".to_string()),
parameters: serde_json::json!({
"type": "object",
"properties": {
"city": {"type": "string"}
},
}),
strict: None,
}]
}
fn test_parser() -> InklingUnifiedParser {
InklingUnifiedParser::new(&test_tools(), Arc::new(FakeTokenizer)).unwrap()
}
fn collect_stream(chunks: &[&str]) -> UnifiedParserOutput {
let mut parser = test_parser();
let mut output = UnifiedParserOutput::default();
for chunk in chunks {
output.append(parser.parse_chunk(chunk).unwrap());
}
output.append(parser.finish().unwrap());
output
}
#[test]
fn inkling_streaming_emits_reasoning_then_text() {
let output = collect_stream(&[concat!(
"<|content_thinking|>reason<|end_message|>",
"<|message_model|><|content_text|>answer<|end_message|>",
"<|content_model_end_sampling|>"
)]);
assert_eq!(output.reasoning_text(), "reason");
assert_eq!(output.normal_text(), "answer");
assert!(output.calls().is_empty());
}
#[test]
fn inkling_streaming_holds_split_markers() {
let output = collect_stream(&[
"<|content_thin",
"king|>rea",
"son<|end_mes",
"sage|><|message_",
"model|><|content_text|>answer",
"<|end_message|>",
]);
assert_eq!(output.reasoning_text(), "reason");
assert_eq!(output.normal_text(), "answer");
}
#[test]
fn inkling_streaming_accepts_content_blocks_without_message_start() {
let output = collect_stream(&[concat!(
"<|content_thinking|>reason<|end_message|>",
"<|content_text|>answer<|end_message|>",
)]);
assert_eq!(output.reasoning_text(), "reason");
assert_eq!(output.normal_text(), "answer");
}
#[test]
fn inkling_tool_json_streams_name_then_argument_deltas() {
let mut parser = test_parser();
let chunks = [
"<|message_model|><|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":",
"{\"city\":",
"\"SF\"",
"}}<|end_message|>",
];
let mut output = UnifiedParserOutput::default();
let mut observed_args = Vec::new();
for chunk in chunks {
let next = parser.parse_chunk(chunk).unwrap();
observed_args.extend(
next.calls()
.into_iter()
.filter(|call| call.name.is_none())
.map(|call| call.arguments),
);
output.append(next);
}
output.append(parser.finish().unwrap());
let calls = output.calls();
assert_eq!(calls[0].name.as_deref(), Some("get_weather"));
assert_eq!(observed_args, ["{\"city\":", "\"SF\"", "}"]);
assert_eq!(
calls.iter().map(|call| call.arguments.as_str()).collect::<String>(),
"{\"city\":\"SF\"}"
);
assert!(output.normal_text().is_empty());
}
#[test]
fn inkling_discards_tool_name_from_message_header() {
let output = collect_stream(&[concat!(
"<|message_model|>get_weather<|content_invoke_tool_json|>",
"{\"name\":\"get_weather\",\"args\":{}}<|end_message|>"
)]);
assert!(output.normal_text().is_empty());
assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather"));
}
#[test]
fn inkling_streaming_handles_multiple_tool_blocks() {
let output = collect_stream(&[concat!(
"<|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":{\"city\":\"SF\"}}<|end_message|>",
"<|message_model|><|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":{\"city\":\"NYC\"}}<|end_message|>"
)]);
let calls = output.calls();
assert_eq!(calls.iter().filter(|call| call.name.is_some()).count(), 2);
assert_eq!(calls[0].tool_index, 0);
assert_eq!(calls[2].tool_index, 1);
}
#[test]
fn inkling_initialize_open_text_prompt_starts_in_text() {
let mut parser = test_parser();
parser.initialize(&[200004]).unwrap();
let output = parser.parse_complete("answer<|end_message|>").unwrap();
assert_eq!(output.normal_text(), "answer");
assert!(output.reasoning_text().is_empty());
}
#[test]
fn inkling_initialize_open_reasoning_prompt_starts_in_reasoning() {
let mut parser = test_parser();
parser.initialize(&[200008]).unwrap();
let output = parser.parse_complete("reason<|end_message|>").unwrap();
assert_eq!(output.reasoning_text(), "reason");
assert!(output.normal_text().is_empty());
}
#[test]
fn inkling_initialize_model_opener_starts_in_message_header() {
let mut parser = test_parser();
parser.initialize(&[200001]).unwrap();
let output = parser
.parse_complete(concat!(
"get_weather<|content_invoke_tool_json|>",
"{\"name\":\"get_weather\",\"args\":{}}<|end_message|>"
))
.unwrap();
assert!(output.normal_text().is_empty());
assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather"));
}
#[test]
fn inkling_plain_text_falls_through_as_text() {
let output = collect_stream(&["plain ", "answer"]);
assert_eq!(output.normal_text(), "plain answer");
assert!(output.reasoning_text().is_empty());
}
#[test]
fn inkling_raw_tool_text_and_tool_error_are_visible_text() {
let output = collect_stream(&[concat!(
"<|content_invoke_tool_text|>search SF<|end_message|>",
"<|content_tool_error|>failed<|end_message|>"
)]);
assert_eq!(output.normal_text(), "search SFfailed");
assert!(output.calls().is_empty());
}
#[test]
fn inkling_finish_fails_incomplete_tool_call() {
let mut parser = test_parser();
parser
.parse_chunk("<|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":{\"city\"")
.unwrap();
let error = parser.finish().unwrap_err();
assert!(error.as_report().to_string().contains("incomplete Inkling tool call"));
}
#[test]
fn inkling_rejects_non_object_args() {
let mut parser = test_parser();
let error = parser
.parse_chunk("<|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":42")
.unwrap_err();
assert!(error.as_report().to_string().contains("JSON object argument"));
}
#[test]
fn inkling_missing_token_fails_create() {
struct MissingTokenizer;
impl Tokenizer for MissingTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(vec![])
}
fn decode(
&self,
_token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(String::new())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
MESSAGE_MODEL => Some(200001),
CONTENT_TEXT => Some(200004),
_ => None,
}
}
fn id_to_token(&self, id: u32) -> Option<String> {
match id {
200001 => Some(MESSAGE_MODEL.to_string()),
200004 => Some(CONTENT_TEXT.to_string()),
_ => None,
}
}
}
let error = match InklingUnifiedParser::new(&[], Arc::new(MissingTokenizer)) {
Ok(_) => panic!("expected parser creation to fail"),
Err(error) => error,
};
assert!(error.as_report().to_string().contains(CONTENT_THINKING));
}
}
+9
View File
@@ -5,9 +5,11 @@
mod combined;
mod gemma4;
mod inkling;
pub use combined::CombinedParser;
pub use gemma4::Gemma4UnifiedParser;
pub use inkling::InklingUnifiedParser;
use thiserror::Error;
use thiserror_ext::Macro;
use vllm_tokenizer::DynTokenizer;
@@ -208,3 +210,10 @@ pub enum UnifiedParserError {
#[error(transparent)]
Tool(#[from] ToolParserError),
}
/// Returns the ID for the given token, or an error if it's not found.
fn token_id(tokenizer: &dyn vllm_tokenizer::Tokenizer, token: &str) -> Result<u32> {
tokenizer.token_to_id(token).ok_or_else(|| UnifiedParserError::MissingToken {
token: token.to_string(),
})
}
+46 -3
View File
@@ -95,6 +95,7 @@ impl HfSpecialTokens {
pub struct ModelConfig {
model_type: Option<String>,
vocab_size: Option<u32>,
eos_token_id: Option<OneOrManyTokenIds>,
num_experts: Option<OneOrManyExpertCount>,
moe_num_experts: Option<OneOrManyExpertCount>,
n_routed_experts: Option<OneOrManyExpertCount>,
@@ -125,12 +126,16 @@ pub(super) enum OneOrManyTokenIds {
}
impl OneOrManyTokenIds {
pub(super) fn into_set(self) -> BTreeSet<u32> {
pub(super) fn as_slice(&self) -> &[u32] {
match self {
Self::One(id) => BTreeSet::from([id]),
Self::Many(ids) => ids.into_iter().collect(),
Self::One(id) => std::slice::from_ref(id),
Self::Many(ids) => ids.as_slice(),
}
}
pub(super) fn into_set(self) -> BTreeSet<u32> {
self.as_slice().iter().copied().collect()
}
}
/// Hugging Face configs may expose the expert count either as one integer or
@@ -195,6 +200,18 @@ impl ModelConfig {
}
}
/// Return the effective model-side EOS token ids, following the same
/// simplified text-config selection as `vocab_size`.
pub(super) fn eos_token_ids(&self) -> &[u32] {
if let Some(eos_token_id) = self.eos_token_id.as_ref() {
eos_token_id.as_slice()
} else if let Some(text_config) = self.text_config.as_deref() {
text_config.eos_token_ids()
} else {
&[]
}
}
/// Match Python's current expert-count priority on the selected text
/// config.
///
@@ -354,6 +371,32 @@ mod tests {
assert_eq!(config.vocab_size().unwrap(), 151936);
}
#[test]
fn model_config_reads_top_level_eos_token_ids() {
let single: ModelConfig = serde_json::from_str(r#"{"eos_token_id":151645}"#).unwrap();
let many: ModelConfig =
serde_json::from_str(r#"{"eos_token_id":[128001,128008,128009]}"#).unwrap();
let null: ModelConfig = serde_json::from_str(r#"{"eos_token_id":null}"#).unwrap();
assert_eq!(single.eos_token_ids(), &[151645]);
assert_eq!(many.eos_token_ids(), &[128001, 128008, 128009]);
assert!(null.eos_token_ids().is_empty());
}
#[test]
fn model_config_uses_nested_eos_token_ids_when_top_level_is_absent() {
let config: ModelConfig = serde_json::from_str(
r#"{
"text_config": {
"eos_token_id": [59246, 59253, 59255]
}
}"#,
)
.unwrap();
assert_eq!(config.eos_token_ids(), &[59246, 59253, 59255]);
}
#[test]
fn model_config_rejects_missing_vocab_size() {
let config: ModelConfig = serde_json::from_str(r#"{}"#).unwrap();
+119 -15
View File
@@ -8,7 +8,9 @@ use std::collections::BTreeSet;
use std::sync::Arc;
use tracing::info;
use vllm_tokenizer::{DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer};
use vllm_tokenizer::{
DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer, Tokenizer,
};
use self::config::{GenerationConfig, load_generation_config};
pub use self::config::{
@@ -56,23 +58,15 @@ impl HfTextBackend {
pub fn from_resolved_model_files(files: ResolvedModelFiles, model_id: String) -> Result<Self> {
let tokenizer_config = load_tokenizer_config(files.tokenizer_config_path.as_deref())?;
let tokenizer = load_tokenizer(&files.tokenizer)?;
let primary_eos_token_id = tokenizer_config
.special_tokens
.eos_token
.as_ref()
.and_then(|token| tokenizer.token_to_id(token.as_str()));
let model_config = load_model_config(files.config_path.as_deref())?;
let model_vocab_size = model_config.vocab_size()? as usize;
let generation_config = load_generation_config(files.generation_config_path.as_deref())?;
let mut extra_eos_token_ids = generation_config
.eos_token_id
.clone()
.map(|value| value.into_set())
.unwrap_or_default();
if let Some(primary_eos_token_id) = primary_eos_token_id {
extra_eos_token_ids.remove(&primary_eos_token_id);
}
let (primary_eos_token_id, extra_eos_token_ids) = resolve_eos_token_ids(
&tokenizer_config,
&model_config,
&generation_config,
tokenizer.as_ref(),
);
info!(
model_id,
@@ -98,6 +92,42 @@ impl HfTextBackend {
}
}
/// Resolve EOS hints from tokenizer, model, and generation configs.
///
/// Resolution rules:
/// 1. Use the tokenizer-side EOS token as the primary EOS when it resolves to a
/// token id.
/// 2. Fall back to the first model-config EOS id when the tokenizer config does
/// not provide a primary EOS.
/// 3. Keep any remaining model/generation EOS ids as extra stop-token ids so
/// they still participate in stopping and min-token handling.
fn resolve_eos_token_ids(
tokenizer_config: &HfTokenizerConfig,
model_config: &ModelConfig,
generation_config: &GenerationConfig,
tokenizer: &dyn Tokenizer,
) -> (Option<u32>, BTreeSet<u32>) {
let model_config_eos_token_ids = model_config.eos_token_ids();
let primary_eos_token_id = tokenizer_config
.special_tokens
.eos_token
.as_ref()
.and_then(|token| tokenizer.token_to_id(token.as_str()))
.or_else(|| model_config_eos_token_ids.first().copied());
let mut extra_eos_token_ids = generation_config
.eos_token_id
.clone()
.map(|value| value.into_set())
.unwrap_or_default();
extra_eos_token_ids.extend(model_config_eos_token_ids.iter().copied());
if let Some(primary_eos_token_id) = primary_eos_token_id {
extra_eos_token_ids.remove(&primary_eos_token_id);
}
(primary_eos_token_id, extra_eos_token_ids)
}
impl TextBackend for HfTextBackend {
fn tokenizer(&self) -> DynTokenizer {
self.tokenizer.clone()
@@ -128,3 +158,77 @@ impl TextBackend for HfTextBackend {
})
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use super::{GenerationConfig, HfTokenizerConfig, ModelConfig, resolve_eos_token_ids};
use vllm_tokenizer::Tokenizer;
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(vec![])
}
fn decode(
&self,
_token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(String::new())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
(token == "</s>").then_some(2)
}
fn id_to_token(&self, id: u32) -> Option<String> {
(id == 2).then(|| "</s>".to_string())
}
}
#[test]
fn eos_resolution_uses_model_config_when_tokenizer_has_no_eos() {
let tokenizer_config: HfTokenizerConfig = serde_json::from_str("{}").unwrap();
let model_config: ModelConfig =
serde_json::from_str(r#"{"eos_token_id":[200006,200010]}"#).unwrap();
let generation_config: GenerationConfig = serde_json::from_str("{}").unwrap();
let (primary, extra) = resolve_eos_token_ids(
&tokenizer_config,
&model_config,
&generation_config,
&FakeTokenizer,
);
assert_eq!(primary, Some(200006));
assert_eq!(extra, BTreeSet::from([200010]));
}
#[test]
fn eos_resolution_keeps_tokenizer_eos_primary() {
let tokenizer_config: HfTokenizerConfig =
serde_json::from_str(r#"{"eos_token":"</s>"}"#).unwrap();
let model_config: ModelConfig =
serde_json::from_str(r#"{"eos_token_id":[2,200006]}"#).unwrap();
let generation_config: GenerationConfig =
serde_json::from_str(r#"{"eos_token_id":[2,200010]}"#).unwrap();
let (primary, extra) = resolve_eos_token_ids(
&tokenizer_config,
&model_config,
&generation_config,
&FakeTokenizer,
);
assert_eq!(primary, Some(2));
assert_eq!(extra, BTreeSet::from([200006, 200010]));
}
}