[Model] Add Inkling model support [1/N] (#48799)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Woosuk Kwon
2026-07-15 23:40:07 -07:00
committed by GitHub
co-authored by Bugen Zhao Giancarlo Delfin Isotr0py Isotr0py Jee Jee Li Roger Wang Yifan Qiao Claude Fable 5 OpenAI Codex
parent 8bfd683901
commit 6570c9800c
95 changed files with 12137 additions and 62 deletions
+2
View File
@@ -20,6 +20,7 @@ use crate::output::{
use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
use crate::renderer::{
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer,
InklingChatRenderer,
};
use crate::request::ChatRequest;
use crate::{DynChatOutputProcessor, RendererSelection};
@@ -71,6 +72,7 @@ impl HfChatBackend {
RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()),
RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
};
info!(
+3 -3
View File
@@ -33,7 +33,7 @@ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
pub use renderer::hf::ChatTemplateContentFormatOption;
pub use renderer::{
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
HarmonyChatRenderer, RenderedPrompt, RendererSelection,
HarmonyChatRenderer, InklingChatRenderer, RenderedPrompt, RendererSelection,
};
pub use request::{
ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
@@ -294,7 +294,7 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string());
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string());
}
#[test]
@@ -305,6 +305,6 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
}
}
+112
View File
@@ -79,6 +79,38 @@ mod tests {
use super::*;
const AUDIO_PAD_ID: u32 = 151_676;
const INKLING_AUDIO_MARKER_ID: u32 = 200_020;
const INKLING_AUDIO_EMBED_ID: i32 = 200_053;
fn inkling_info(decoder_dmodel: serde_json::Value) -> MultimodalModelInfo {
let config = serde_json::json!({
"model_type": "inkling_mm_model",
"audio_config": {
"decoder_dmodel": decoder_dmodel,
"n_mel_bins": 80,
"mel_vocab_size": 16,
"dmel_min_value": -7.0,
"dmel_max_value": 2.0
}
});
let tokenizer = TestTokenizer::new()
.with_regular_token("<|content_image|>", 200_005)
.with_regular_token("<|content_audio_input|>", INKLING_AUDIO_MARKER_ID);
let context = MultimodalModelContext {
model_id: "inkling-test".to_string(),
model_type: Some("inkling_mm_model".to_string()),
config,
tokenizer: TokenizerResolver(Arc::new(tokenizer)),
};
MultimodalModelInfo::from_loaded(
context,
PreProcessorConfig::default(),
PreProcessorConfig::default(),
)
.unwrap()
.expect("Inkling multimodal support")
}
fn qwen3_asr_info() -> MultimodalModelInfo {
let context = MultimodalModelContext {
@@ -120,6 +152,40 @@ mod tests {
bytes
}
#[test]
fn resolves_inkling_audio_from_model_spec() {
let info = inkling_info(serde_json::json!(1024));
let support = info.audio.as_ref().expect("audio support");
assert_eq!(
info.placeholder_token(Modality::Audio),
Some("<|content_audio_input|>")
);
assert_eq!(support.placeholder.marker_token_id, INKLING_AUDIO_MARKER_ID);
assert_eq!(
support.placeholder.embed_token_id,
INKLING_AUDIO_EMBED_ID as u32
);
assert_eq!(support.spec.primary_key(), AUDIO_PRIMARY_KEY);
assert!(matches!(
&support.spec.field_layouts.encoder_input,
llm_multimodal::FieldLayout::Flat { sizes_key }
if sizes_key == "num_audio_tokens"
));
assert!(matches!(
support.spec.field_layouts.model_specific.get("num_audio_tokens"),
Some(llm_multimodal::FieldLayout::Batched)
));
}
#[test]
fn inkling_decoder_config_gates_audio_capability() {
let info = inkling_info(serde_json::Value::Null);
assert!(info.audio.is_none());
assert_eq!(info.placeholder_token(Modality::Audio), None);
}
#[test]
fn resolves_qwen_audio_from_model_spec() {
let info = qwen3_asr_info();
@@ -210,4 +276,50 @@ mod tests {
if tensor.dtype == "int64" && tensor.shape.is_empty()
));
}
#[tokio::test]
async fn inkling_tracker_and_processor_use_standard_audio_key() {
let info = inkling_info(serde_json::json!(1024));
let wav = wav_i16_mono(16_000, &[0; 1_600]);
let expected_hash = llm_multimodal::hasher::hash_audio(&wav);
let fetched = info
.fetch_media(vec![MediaContentPart::AudioData {
data: wav,
mime_type: Some("audio/wav".to_string()),
uuid: Some("audio-1".to_string()),
}])
.await
.unwrap();
let prepared = info.prepare_audios(fetched.audios, fetched.audio_uuids).await.unwrap();
assert_eq!(prepared.replacements.len(), 1);
assert_eq!(
prepared.replacements[0].tokens[0],
INKLING_AUDIO_MARKER_ID as i32
);
assert!(
prepared.replacements[0].tokens[1..]
.iter()
.all(|token| *token == INKLING_AUDIO_EMBED_ID)
);
let item = &prepared.items[0];
assert_eq!(item.hash, expected_hash);
assert_eq!(item.uuid.as_deref(), Some("audio-1"));
let features = &item.data[AUDIO_PRIMARY_KEY];
assert!(matches!(&features.field, MmField::Flat(_)));
assert!(matches!(
features.data.as_ref(),
Some(MmKwargValue::Tensor(tensor))
if tensor.dtype == "float32" && tensor.shape.get(1) == Some(&80)
));
let count = &item.data["num_audio_tokens"];
assert!(matches!(&count.field, MmField::Batched(_)));
assert!(matches!(
count.data.as_ref(),
Some(MmKwargValue::Tensor(tensor))
if tensor.dtype == "int64" && tensor.shape.is_empty()
));
}
}
@@ -23,6 +23,7 @@ pub mod names {
pub const DEEPSEEK_V3: &str = "deepseek_v3";
pub const DEEPSEEK_V4: &str = "deepseek_v4";
pub const GEMMA4: &str = "gemma4";
pub const INKLING: &str = "inkling";
pub const GLM45: &str = "glm45";
pub const KIMI: &str = "kimi";
pub const KIMI_K2: &str = "kimi_k2";
@@ -63,6 +64,7 @@ impl ReasoningParserFactory {
.register_parser::<DeepSeekV3ReasoningParser>(names::DEEPSEEK_V3)
.register_parser::<DeepSeekV4ReasoningParser>(names::DEEPSEEK_V4)
.register_unified_dummy(names::GEMMA4)
.register_unified_dummy(names::INKLING)
.register_parser::<Glm45ReasoningParser>(names::GLM45)
.register_parser::<KimiReasoningParser>(names::KIMI)
.register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
+2
View File
@@ -25,6 +25,7 @@ pub mod names {
pub const GLM45: &str = "glm45";
pub const GLM47: &str = "glm47";
pub const GEMMA4: &str = "gemma4";
pub const INKLING: &str = "inkling";
pub const GRANITE4: &str = "granite4";
pub const HERMES: &str = "hermes";
pub const HY_V3: &str = "hy_v3";
@@ -70,6 +71,7 @@ impl ToolParserFactory {
.register_parser::<Glm45MoeToolParser>(names::GLM45)
.register_parser::<Glm47MoeToolParser>(names::GLM47)
.register_unified_dummy(names::GEMMA4)
.register_unified_dummy(names::INKLING)
.register_parser::<Granite4ToolParser>(names::GRANITE4)
.register_parser::<HermesToolParser>(names::HERMES)
.register_parser::<HyV3ToolParser>(names::HY_V3)
+24 -2
View File
@@ -5,7 +5,7 @@
use std::sync::LazyLock;
pub use vllm_parser::unified::{Gemma4UnifiedParser, UnifiedParser};
pub use vllm_parser::unified::{Gemma4UnifiedParser, InklingUnifiedParser, UnifiedParser};
use vllm_tokenizer::DynTokenizer;
use crate::parser::ParserFactory;
@@ -14,6 +14,7 @@ use crate::request::ChatTool;
/// Canonical public names for registered unified parsers.
pub mod names {
pub const GEMMA4: &str = "gemma4";
pub const INKLING: &str = "inkling";
}
/// Constructor signature for one registered unified parser implementation.
@@ -37,10 +38,12 @@ impl UnifiedParserFactory {
let mut factory = Self::default();
factory.register_parser::<Gemma4UnifiedParser>(names::GEMMA4);
factory.register_parser::<InklingUnifiedParser>(names::INKLING);
factory
.register_pattern("gemma-4", names::GEMMA4)
.register_pattern("gemma4", names::GEMMA4);
.register_pattern("gemma4", names::GEMMA4)
.register_pattern("inkling", names::INKLING);
factory
}
@@ -88,6 +91,13 @@ mod tests {
.with_regular_token("<channel|>", 257)
}
fn inkling_tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_regular_token("<|message_model|>", 200001)
.with_regular_token("<|content_text|>", 200004)
.with_regular_token("<|content_thinking|>", 200008)
}
#[test]
fn factory_registers_gemma4() {
let factory = UnifiedParserFactory::new();
@@ -99,4 +109,16 @@ mod tests {
);
factory.create(names::GEMMA4, &[], Arc::new(tokenizer())).unwrap();
}
#[test]
fn factory_registers_inkling() {
let factory = UnifiedParserFactory::new();
assert!(factory.contains(names::INKLING));
assert_eq!(
factory.resolve_name_for_model("thinkingmachines/Inkling"),
Some(names::INKLING)
);
factory.create(names::INKLING, &[], Arc::new(inkling_tokenizer())).unwrap();
}
}
@@ -0,0 +1,21 @@
{
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "transcribe"
},
{
"type": "input_audio",
"data": ""
},
{
"type": "audio_url",
"audio_url": "data:audio/wav;base64,"
}
]
}
]
}
@@ -0,0 +1 @@
<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>transcribe<|end_message|><|message_user|><|content_audio_input|><|audio_end|><|end_message|><|message_user|><|content_audio_input|><|audio_end|><|end_message|><|message_model|>
@@ -0,0 +1,17 @@
{
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "look"
},
{
"type": "image_url",
"image_url": "data:image/png;base64,"
}
]
}
]
}
@@ -0,0 +1 @@
<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>look<|end_message|><|message_user|><|content_image|><|end_message|><|message_model|>
@@ -0,0 +1,46 @@
{
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather information",
"parameters": {
"type": "object",
"required": [
"city"
],
"properties": {
"city": {
"type": "string"
}
}
}
}
}
],
"messages": [
{
"role": "developer",
"content": "rules",
"tools": [
{
"type": "function",
"function": {
"name": "local_tool",
"parameters": {
"z": 1,
"a": {
"b": 2
}
}
}
}
]
},
{
"role": "user",
"content": "hi"
}
]
}
@@ -0,0 +1 @@
<|message_system|>tool_declare<|content_xml|>[{"description":"Get weather information","name":"get_weather","parameters":{"properties":{"city":{"type":"string"}},"required":["city"],"type":"object"},"type":"function"},{"description":"","name":"local_tool","parameters":{"a":{"b":2},"z":1},"type":"function"}]<|end_message|><|message_system|><|content_text|>rules<|end_message|><|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>hi<|end_message|><|message_model|>
@@ -0,0 +1,24 @@
{
"add_generation_prompt": false,
"messages": [
{
"role": "assistant",
"reasoning_content": "think",
"content": "answer",
"tool_calls": [
{
"id": "call_1",
"function": {
"name": "get_weather",
"arguments": "{\"city\":\"SF\"}"
}
}
]
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "sunny"
}
]
}
@@ -0,0 +1 @@
<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_model|><|content_thinking|>think<|end_message|><|message_model|><|content_text|>answer<|end_message|><|message_model|>get_weather<|content_invoke_tool_json|>{"name":"get_weather","args":{"city":"SF"}}<|end_message|><|content_model_end_sampling|><|message_tool|>get_weather<|content_text|>sunny<|end_message|>
+442
View File
@@ -0,0 +1,442 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use serde_json::{Map, Value, json};
use thiserror_ext::AsReport as _;
use vllm_text::Prompt;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
use crate::error::{Error, Result};
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool};
use crate::{AssistantContentBlock, AssistantToolCall};
const MESSAGE_USER: &str = "<|message_user|>";
const MESSAGE_MODEL: &str = "<|message_model|>";
const MESSAGE_SYSTEM: &str = "<|message_system|>";
const MESSAGE_TOOL: &str = "<|message_tool|>";
const CONTENT_TEXT: &str = "<|content_text|>";
const CONTENT_IMAGE: &str = "<|content_image|>";
const CONTENT_MODEL_END_SAMPLING: &str = "<|content_model_end_sampling|>";
const CONTENT_AUDIO_INPUT: &str = "<|content_audio_input|>";
const CONTENT_THINKING: &str = "<|content_thinking|>";
// Inkling renderer semantics use this slot for structured payloads such as tool
// declarations.
const CONTENT_XML: &str = "<|content_xml|>";
const CONTENT_INVOKE_TOOL_JSON: &str = "<|content_invoke_tool_json|>";
const END_MESSAGE: &str = "<|end_message|>";
const AUDIO_END: &str = "<|audio_end|>";
const MAX_REASONING_EFFORT: f64 = 0.99;
/// Native Inkling renderer that emits token IDs directly.
#[derive(Clone)]
pub struct InklingChatRenderer {
tokenizer: DynTokenizer,
special: InklingSpecialTokenIds,
}
#[derive(Debug, Clone, Copy)]
struct InklingSpecialTokenIds {
message_user: u32,
message_model: u32,
message_system: u32,
message_tool: u32,
content_text: u32,
content_image: u32,
content_model_end_sampling: u32,
content_audio_input: u32,
content_thinking: u32,
content_xml: u32,
content_invoke_tool_json: u32,
end_message: u32,
audio_end: u32,
}
impl InklingSpecialTokenIds {
fn resolve(tokenizer: &dyn Tokenizer) -> Result<Self> {
Ok(Self {
message_user: resolve_special_token(tokenizer, MESSAGE_USER)?,
message_model: resolve_special_token(tokenizer, MESSAGE_MODEL)?,
message_system: resolve_special_token(tokenizer, MESSAGE_SYSTEM)?,
message_tool: resolve_special_token(tokenizer, MESSAGE_TOOL)?,
content_text: resolve_special_token(tokenizer, CONTENT_TEXT)?,
content_image: resolve_special_token(tokenizer, CONTENT_IMAGE)?,
content_model_end_sampling: resolve_special_token(
tokenizer,
CONTENT_MODEL_END_SAMPLING,
)?,
content_audio_input: resolve_special_token(tokenizer, CONTENT_AUDIO_INPUT)?,
content_thinking: resolve_special_token(tokenizer, CONTENT_THINKING)?,
content_xml: resolve_special_token(tokenizer, CONTENT_XML)?,
content_invoke_tool_json: resolve_special_token(tokenizer, CONTENT_INVOKE_TOOL_JSON)?,
end_message: resolve_special_token(tokenizer, END_MESSAGE)?,
audio_end: resolve_special_token(tokenizer, AUDIO_END)?,
})
}
}
impl InklingChatRenderer {
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
let special = InklingSpecialTokenIds::resolve(tokenizer.as_ref())?;
Ok(Self { tokenizer, special })
}
fn write_text_tokens(&self, out: &mut Vec<u32>, text: &str) -> Result<()> {
out.extend(self.tokenizer.encode(text, false)?);
Ok(())
}
fn write_message_start(
&self,
out: &mut Vec<u32>,
role_token_id: u32,
author_name: Option<&str>,
) -> Result<()> {
out.push(role_token_id);
if let Some(author_name) = author_name
&& !author_name.is_empty()
{
self.write_text_tokens(out, author_name)?;
}
Ok(())
}
fn write_text_block(
&self,
out: &mut Vec<u32>,
role_token_id: u32,
author_name: Option<&str>,
text: &str,
) -> Result<()> {
self.write_message_start(out, role_token_id, author_name)?;
out.push(self.special.content_text);
self.write_text_tokens(out, text)?;
out.push(self.special.end_message);
Ok(())
}
/// Write one image block holding only the `<|content_image|>` marker.
/// Multimodal preprocessing later expands the marker into per-patch
/// image placeholder tokens once the patch count is known, mirroring the
/// Python `InklingMultiModalProcessor` marker-anchored prompt updates.
fn write_image_block(&self, out: &mut Vec<u32>, role_token_id: u32) {
out.push(role_token_id);
out.push(self.special.content_image);
out.push(self.special.end_message);
}
/// Write one audio block holding the `<|content_audio_input|>` marker
/// followed by the `<|audio_end|>` terminator. Multimodal preprocessing
/// later expands the marker into per-frame audio placeholder tokens
/// (landing before `<|audio_end|>`) once the clip length is known.
fn write_audio_block(&self, out: &mut Vec<u32>, role_token_id: u32) {
out.push(role_token_id);
out.push(self.special.content_audio_input);
out.push(self.special.audio_end);
out.push(self.special.end_message);
}
fn write_reasoning_block(&self, out: &mut Vec<u32>, text: &str) -> Result<()> {
if text.is_empty() {
return Ok(());
}
out.push(self.special.message_model);
out.push(self.special.content_thinking);
self.write_text_tokens(out, text)?;
out.push(self.special.end_message);
Ok(())
}
fn write_reasoning_effort(&self, out: &mut Vec<u32>, effort: f64) -> Result<()> {
if !(0.0..=MAX_REASONING_EFFORT).contains(&effort) {
return Err(Error::ChatTemplate(format!(
"Inkling reasoning_effort must be in [0.0, 0.99], got {effort}"
)));
}
let formatted = format!("{effort:.2}");
let effort = formatted.trim_end_matches('0').trim_end_matches('.');
let effort = if matches!(effort, "0" | "-0") {
"0.0"
} else {
effort
};
self.write_text_block(
out,
self.special.message_system,
None,
&format!("Thinking effort level: {effort}"),
)
}
fn write_tool_declarations(&self, out: &mut Vec<u32>, tools: &[&ChatTool]) -> Result<()> {
if tools.is_empty() {
return Ok(());
}
let mut specs = Vec::with_capacity(tools.len());
for tool in tools {
specs.push(json!({
"description": tool.description.as_deref().unwrap_or(""),
"name": tool.name,
"parameters": sort_json(&tool.parameters),
"type": "function",
}));
}
let payload = compact_json(&sort_json(&Value::Array(specs)))?;
self.write_message_start(out, self.special.message_system, Some("tool_declare"))?;
out.push(self.special.content_xml);
self.write_text_tokens(out, &payload)?;
out.push(self.special.end_message);
Ok(())
}
fn write_chat_content(
&self,
out: &mut Vec<u32>,
role_token_id: u32,
content: &ChatContent,
) -> Result<()> {
match content {
ChatContent::Text(text) => {
if !text.is_empty() {
self.write_text_block(out, role_token_id, None, text)?;
}
}
ChatContent::Parts(parts) => {
for part in parts {
match part {
ChatContentPart::Text { text } => {
self.write_text_block(out, role_token_id, None, text)?;
}
ChatContentPart::ImageUrl { .. } => {
self.write_image_block(out, role_token_id);
}
ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => {
self.write_audio_block(out, role_token_id);
}
// Inkling has no video modality.
ChatContentPart::VideoUrl { .. } => {
return Err(Error::UnsupportedMultimodalContent("video_url"));
}
}
}
}
}
Ok(())
}
fn write_assistant_tool_call(
&self,
out: &mut Vec<u32>,
tool_call: &AssistantToolCall,
) -> Result<()> {
let payload = tool_call_json(tool_call)?;
self.write_message_start(out, self.special.message_model, Some(&tool_call.name))?;
out.push(self.special.content_invoke_tool_json);
self.write_text_tokens(out, &payload)?;
out.push(self.special.end_message);
Ok(())
}
fn write_assistant_content(
&self,
out: &mut Vec<u32>,
content: &[AssistantContentBlock],
tool_call_id_to_name: &mut HashMap<String, String>,
) -> Result<()> {
for block in content {
match block {
AssistantContentBlock::Reasoning { text } => {
self.write_reasoning_block(out, text)?;
}
AssistantContentBlock::Text { text } => {
if !text.is_empty() {
self.write_text_block(out, self.special.message_model, None, text)?;
}
}
AssistantContentBlock::ToolCall(tool_call) => {
if !tool_call.id.is_empty() {
tool_call_id_to_name.insert(tool_call.id.clone(), tool_call.name.clone());
}
self.write_assistant_tool_call(out, tool_call)?;
}
}
}
out.push(self.special.content_model_end_sampling);
Ok(())
}
fn write_tool_response(
&self,
out: &mut Vec<u32>,
content: &ChatContent,
tool_call_id: &str,
tool_call_id_to_name: &HashMap<String, String>,
) -> Result<()> {
let text = content.try_flatten_to_text()?;
let tool_name = tool_call_id_to_name.get(tool_call_id).map(String::as_str).unwrap_or("");
self.write_text_block(out, self.special.message_tool, Some(tool_name), &text)
}
}
impl ChatRenderer for InklingChatRenderer {
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
request.validate()?;
if request.chat_options.continue_final_message() {
return Err(Error::ChatTemplate(
"Inkling renderer does not support continue_final_message".to_string(),
));
}
let mut out = Vec::new();
let mut tool_call_id_to_name = HashMap::new();
let effective_template_kwargs = request_template_kwargs(request);
let tools = rendered_tools(request);
self.write_tool_declarations(&mut out, &tools)?;
let mut reasoning_effort =
resolve_reasoning_effort(effective_template_kwargs.get("reasoning_effort"));
for message in &request.messages {
if !matches!(
message,
ChatMessage::System { .. } | ChatMessage::Developer { .. }
) && let Some(effort) = reasoning_effort.take()
{
self.write_reasoning_effort(&mut out, effort)?;
}
match message {
ChatMessage::System { content } => {
self.write_chat_content(&mut out, self.special.message_system, content)?;
}
ChatMessage::Developer { content, .. } => {
self.write_chat_content(&mut out, self.special.message_system, content)?;
}
ChatMessage::User { content } => {
self.write_chat_content(&mut out, self.special.message_user, content)?;
}
ChatMessage::Assistant { content } => {
self.write_assistant_content(&mut out, content, &mut tool_call_id_to_name)?;
}
ChatMessage::ToolResponse {
content,
tool_call_id,
} => {
self.write_tool_response(
&mut out,
content,
tool_call_id,
&tool_call_id_to_name,
)?;
}
}
}
if let Some(effort) = reasoning_effort {
self.write_reasoning_effort(&mut out, effort)?;
}
if request.chat_options.add_generation_prompt() {
out.push(self.special.message_model);
}
Ok(RenderedPrompt {
prompt: Prompt::TokenIds(out),
effective_template_kwargs,
})
}
}
fn resolve_reasoning_effort(value: Option<&Value>) -> Option<f64> {
let Some(value) = value else {
return Some(0.9);
};
match value {
Value::String(name) => match name.as_str() {
"none" => Some(0.0),
"minimal" => Some(0.1),
"low" => Some(0.2),
"medium" => Some(0.7),
"high" => Some(0.9),
"xhigh" | "max" => Some(0.99),
_ => None,
},
Value::Number(number) => number.as_f64(),
_ => None,
}
}
fn resolve_special_token(tokenizer: &dyn Tokenizer, token: &str) -> Result<u32> {
tokenizer.token_to_id(token).ok_or_else(|| {
Error::ChatTemplate(format!(
"Inkling tokenizer is missing special token `{token}`"
))
})
}
fn rendered_tools(request: &ChatRequest) -> Vec<&ChatTool> {
if !request.tool_parsing_enabled() {
return Vec::new();
}
let mut tools = Vec::with_capacity(request.tools.len());
tools.extend(request.tools.iter());
for message in &request.messages {
if let ChatMessage::Developer {
tools: Some(local_tools),
..
} = message
{
tools.extend(local_tools.iter());
}
}
tools
}
fn tool_call_json(tool_call: &AssistantToolCall) -> Result<String> {
let name_json = serde_json::to_string(&tool_call.name)
.map_err(|error| Error::ChatTemplate(error.as_report().to_string()))?;
let arguments = if tool_call.arguments.trim().is_empty() {
Value::Object(Map::new())
} else {
serde_json::from_str(&tool_call.arguments).map_err(|error| {
Error::ChatTemplate(format!(
"Inkling tool call arguments must decode to a JSON object: {error}"
))
})?
};
let Value::Object(_) = arguments else {
return Err(Error::ChatTemplate(
"Inkling tool call arguments must decode to a JSON object".to_string(),
));
};
let args_json = compact_json(&sort_json(&arguments))?;
Ok(format!("{{\"name\":{name_json},\"args\":{args_json}}}"))
}
fn compact_json(value: &Value) -> Result<String> {
serde_json::to_string(value).map_err(|error| Error::ChatTemplate(error.as_report().to_string()))
}
fn sort_json(value: &Value) -> Value {
match value {
Value::Array(items) => Value::Array(items.iter().map(sort_json).collect()),
Value::Object(map) => {
let mut sorted = Map::new();
let mut keys = map.keys().collect::<Vec<_>>();
keys.sort();
for key in keys {
sorted.insert(key.clone(), sort_json(&map[key]));
}
Value::Object(sorted)
}
_ => value.clone(),
}
}
#[cfg(test)]
mod tests;
+375
View File
@@ -0,0 +1,375 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::path::PathBuf;
use std::sync::Arc;
use expect_test::{ExpectFile, expect_file};
use serde_json::json;
use thiserror_ext::AsReport;
use vllm_text::tokenizer::Tokenizer;
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
use super::{
AUDIO_END, CONTENT_AUDIO_INPUT, CONTENT_IMAGE, CONTENT_INVOKE_TOOL_JSON, CONTENT_TEXT,
CONTENT_THINKING, CONTENT_XML, END_MESSAGE, InklingChatRenderer, MESSAGE_MODEL, MESSAGE_SYSTEM,
MESSAGE_TOOL, MESSAGE_USER,
};
use crate::event::{AssistantContentBlock, AssistantToolCall};
use crate::request::{ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort};
use crate::{ChatRenderer, Error};
struct FixtureTokenizer;
impl Tokenizer for FixtureTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
Ok(text.bytes().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<|message_user|>" => Some(200000),
"<|message_model|>" => Some(200001),
"<|message_system|>" => Some(200002),
"<|message_tool|>" => Some(200003),
"<|content_text|>" => Some(200004),
"<|content_image|>" => Some(200005),
"<|content_model_end_sampling|>" => Some(200006),
"<|content_thinking|>" => Some(200008),
"<|end_message|>" => Some(200010),
"<|content_audio_input|>" => Some(200020),
CONTENT_XML => Some(200024),
"<|audio_end|>" => Some(200043),
"<|content_invoke_tool_json|>" => Some(200049),
_ => None,
}
}
fn id_to_token(&self, id: u32) -> Option<String> {
let token = match id {
200000 => "<|message_user|>",
200001 => "<|message_model|>",
200002 => "<|message_system|>",
200003 => "<|message_tool|>",
200004 => "<|content_text|>",
200005 => "<|content_image|>",
200006 => "<|content_model_end_sampling|>",
200008 => "<|content_thinking|>",
200010 => "<|end_message|>",
200020 => "<|content_audio_input|>",
200024 => CONTENT_XML,
200043 => "<|audio_end|>",
200049 => "<|content_invoke_tool_json|>",
_ => return None,
};
Some(token.to_string())
}
}
fn renderer() -> InklingChatRenderer {
InklingChatRenderer::new(Arc::new(FixtureTokenizer)).unwrap()
}
fn render_token_ids(request: &ChatRequest) -> Vec<u32> {
renderer()
.render(request)
.unwrap()
.prompt
.into_token_ids()
.expect("Inkling renderer returns token IDs")
}
fn fixture_request(name: &str) -> ChatRequest {
fixture_chat_request(&fixture_path(name), inkling_fixture_options())
}
fn inkling_fixture_options() -> FixtureRequestOptions {
FixtureRequestOptions {
enable_thinking: false,
no_generation_prompt_when_last_assistant: false,
}
}
fn fixture_path(name: &str) -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("src/renderer/inkling")
.join("fixtures")
.join(name)
}
fn assert_fixture(input_name: &str, expected: ExpectFile) {
let request = fixture_request(input_name);
let rendered = format!("{}\n", render_symbolic_tokens(&render_token_ids(&request)));
expected.assert_eq(&rendered);
}
fn render_symbolic_tokens(token_ids: &[u32]) -> String {
let mut out = String::new();
let mut text_bytes = Vec::new();
for token_id in token_ids {
if let Some(marker) = symbolic_tokens()
.iter()
.find_map(|(marker, marker_id)| (token_id == marker_id).then_some(*marker))
{
flush_text_bytes(&mut out, &mut text_bytes);
out.push_str(marker);
continue;
}
let byte = u8::try_from(*token_id)
.unwrap_or_else(|_| panic!("unexpected non-byte fixture text token {token_id}"));
text_bytes.push(byte);
}
flush_text_bytes(&mut out, &mut text_bytes);
out
}
fn flush_text_bytes(out: &mut String, text_bytes: &mut Vec<u8>) {
if text_bytes.is_empty() {
return;
}
out.push_str(std::str::from_utf8(text_bytes).unwrap());
text_bytes.clear();
}
fn symbolic_tokens() -> [(&'static str, u32); 13] {
[
(MESSAGE_USER, 200000),
(MESSAGE_MODEL, 200001),
(MESSAGE_SYSTEM, 200002),
(MESSAGE_TOOL, 200003),
(CONTENT_TEXT, 200004),
(CONTENT_IMAGE, 200005),
(super::CONTENT_MODEL_END_SAMPLING, 200006),
(CONTENT_THINKING, 200008),
(END_MESSAGE, 200010),
(CONTENT_AUDIO_INPUT, 200020),
(CONTENT_XML, 200024),
(AUDIO_END, 200043),
(CONTENT_INVOKE_TOOL_JSON, 200049),
]
}
#[test]
fn renders_text_and_image_as_inkling_tokens() {
assert_fixture(
"text_image_input.json",
expect_file!["fixtures/text_image_output.txt"],
);
}
#[test]
fn renders_audio_marker_only_blocks() {
assert_fixture(
"text_audio_input.json",
expect_file!["fixtures/text_audio_output.txt"],
);
}
#[test]
fn renders_reasoning_text_tool_call_and_tool_response() {
assert_fixture(
"tool_round_trip_input.json",
expect_file!["fixtures/tool_round_trip_output.txt"],
);
}
#[test]
fn renders_request_and_developer_tools_as_tool_declare() {
assert_fixture(
"tool_declare_input.json",
expect_file!["fixtures/tool_declare_output.txt"],
);
}
#[test]
fn renders_named_reasoning_effort_after_tool_declarations() {
for (effort, expected) in [
(ReasoningEffort::None, "0.0"),
(ReasoningEffort::Minimal, "0.1"),
(ReasoningEffort::Low, "0.2"),
(ReasoningEffort::Medium, "0.7"),
(ReasoningEffort::High, "0.9"),
(ReasoningEffort::XHigh, "0.99"),
(ReasoningEffort::Max, "0.99"),
] {
let mut request = fixture_request("tool_declare_input.json");
request.chat_options.reasoning_effort = Some(effort);
let rendered = render_symbolic_tokens(&render_token_ids(&request));
let tool_end = rendered.find("<|end_message|>").unwrap();
let effort_block = format!(
"<|message_system|><|content_text|>Thinking effort level: \
{expected}<|end_message|>"
);
let effort_start = rendered.find(&effort_block).unwrap();
let system_end = rendered.find("rules<|end_message|>").unwrap();
assert!(effort_start > tool_end);
assert!(effort_start > system_end);
}
}
#[test]
fn emits_one_reasoning_effort_for_multi_turn_conversation() {
let request = ChatRequest {
messages: vec![
ChatMessage::system("rules"),
ChatMessage::user("user1"),
ChatMessage::assistant_text("assistant1"),
ChatMessage::user("user2"),
],
..ChatRequest::for_test()
};
let rendered = render_symbolic_tokens(&render_token_ids(&request));
let effort = "<|message_system|><|content_text|>Thinking effort level: 0.9\
<|end_message|>";
assert_eq!(rendered.matches(effort).count(), 1);
assert!(rendered.find("rules").unwrap() < rendered.find(effort).unwrap());
assert!(rendered.find(effort).unwrap() < rendered.find("user1").unwrap());
assert!(rendered.find("user1").unwrap() < rendered.find("assistant1").unwrap());
assert!(rendered.find("assistant1").unwrap() < rendered.find("user2").unwrap());
}
#[test]
fn canonicalizes_numeric_zero_reasoning_effort() {
for value in [0.0, -0.0] {
let mut request = ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("reasoning_effort".to_string(), json!(value));
assert!(
render_symbolic_tokens(&render_token_ids(&request)).starts_with(
"<|message_system|><|content_text|>Thinking effort level: 0.0\
<|end_message|>"
)
);
}
}
#[test]
fn defaults_reasoning_effort_to_high() {
let request = ChatRequest::for_test();
assert!(
render_symbolic_tokens(&render_token_ids(&request)).starts_with(
"<|message_system|><|content_text|>Thinking effort level: 0.9\
<|end_message|>"
)
);
}
#[test]
fn renders_numeric_reasoning_effort_template_kwarg() {
let mut request = ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("reasoning_effort".to_string(), json!(0.8));
assert_eq!(
render_symbolic_tokens(&render_token_ids(&request)),
"<|message_system|><|content_text|>Thinking effort level: 0.8\
<|end_message|><|message_user|><|content_text|>test\
<|end_message|><|message_model|>"
);
}
#[test]
fn ignores_unsupported_reasoning_effort_values() {
for value in [json!(true), json!("invalid"), json!(null)] {
let mut request = ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("reasoning_effort".to_string(), value);
assert_eq!(
render_symbolic_tokens(&render_token_ids(&request)),
"<|message_user|><|content_text|>test<|end_message|><|message_model|>"
);
}
}
#[test]
fn rejects_out_of_range_reasoning_effort() {
for value in [0.990_000_1, 1.0, 1.5, -0.1] {
let mut request = ChatRequest::for_test();
request
.chat_options
.template_kwargs
.insert("reasoning_effort".to_string(), json!(value));
let error = renderer().render(&request).unwrap_err();
assert!(error.as_report().to_string().contains("must be in [0.0, 0.99]"));
}
}
#[test]
fn rejects_continue_final_message() {
let request = ChatRequest {
messages: vec![ChatMessage::assistant_text("partial")],
chat_options: crate::ChatOptions {
generation_prompt_mode: GenerationPromptMode::ContinueFinalAssistant,
..Default::default()
},
..ChatRequest::for_test()
};
let error = renderer().render(&request).unwrap_err();
assert!(matches!(error, Error::ChatTemplate(_)));
assert!(error.as_report().to_string().contains("continue_final_message"));
}
#[test]
fn renders_developer_messages_as_system() {
let request = ChatRequest {
messages: vec![ChatMessage::developer("rules", None)],
..ChatRequest::for_test()
};
assert_eq!(
render_symbolic_tokens(&render_token_ids(&request)),
"<|message_system|><|content_text|>rules<|end_message|>\
<|message_system|><|content_text|>Thinking effort level: 0.9\
<|end_message|><|message_model|>"
);
}
#[test]
fn rejects_non_object_tool_call_arguments() {
let request = ChatRequest {
messages: vec![ChatMessage::assistant_blocks(vec![
AssistantContentBlock::ToolCall(AssistantToolCall {
id: "call_1".to_string(),
name: "get_weather".to_string(),
arguments: "[]".to_string(),
}),
])],
..ChatRequest::for_test()
};
let error = renderer().render(&request).unwrap_err();
assert!(error.as_report().to_string().contains("JSON object"));
}
+2
View File
@@ -14,6 +14,7 @@ pub mod deepseek_v32;
pub mod deepseek_v4;
pub mod harmony;
pub mod hf;
mod inkling;
mod selection;
#[cfg(test)]
mod test_utils;
@@ -21,6 +22,7 @@ mod test_utils;
pub use deepseek_v4::DeepSeekV4ChatRenderer;
pub use deepseek_v32::DeepSeekV32ChatRenderer;
pub use harmony::HarmonyChatRenderer;
pub use inkling::InklingChatRenderer;
pub use selection::RendererSelection;
/// Rendered chat prompt submitted to the text backend.
+9 -1
View File
@@ -24,6 +24,8 @@ pub enum RendererSelection {
DeepSeekV4,
/// Force the GPT-OSS Harmony renderer.
Harmony,
/// Force the Inkling native token renderer.
Inkling,
}
impl RendererSelection {
@@ -33,6 +35,8 @@ impl RendererSelection {
pub const GPT_OSS_MODEL_TYPE: &str = "gpt_oss";
pub const HARMONY_LITERAL: &str = "harmony";
pub const HF_LITERAL: &str = "hf";
pub const INKLING_LITERAL: &str = "inkling";
pub const INKLING_MODEL_TYPE: &str = "inkling_mm_model";
/// Resolve the renderer selection using the given model type string, if
/// it's `Auto`.
@@ -42,6 +46,7 @@ impl RendererSelection {
Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32,
Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
Self::GPT_OSS_MODEL_TYPE => Self::Harmony,
Self::INKLING_MODEL_TYPE => Self::Inkling,
_ => Self::Hf,
},
selection => selection,
@@ -63,6 +68,8 @@ impl FromStr for RendererSelection {
Ok(Self::DeepSeekV4)
} else if value.eq_ignore_ascii_case(Self::HARMONY_LITERAL) {
Ok(Self::Harmony)
} else if value.eq_ignore_ascii_case(Self::INKLING_LITERAL) {
Ok(Self::Inkling)
} else {
Err(format!(
"unknown renderer `{value}` (expected one of: {})",
@@ -80,6 +87,7 @@ impl fmt::Display for RendererSelection {
Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL),
Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
Self::Harmony => f.write_str(Self::HARMONY_LITERAL),
Self::Inkling => f.write_str(Self::INKLING_LITERAL),
}
}
}
@@ -106,7 +114,7 @@ mod tests {
fn renderer_selection_expected_error_message() {
let err = RendererSelection::from_str("unknown").unwrap_err();
expect_test::expect![
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)"
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling)"
]
.assert_eq(&err);
}
+11
View File
@@ -103,6 +103,8 @@ pub(crate) enum FixtureContent {
pub(crate) enum FixtureContentPart {
Text { text: String },
ImageUrl { image_url: String },
InputAudio { data: String },
AudioUrl { audio_url: String },
}
#[derive(Debug, Deserialize)]
@@ -226,6 +228,15 @@ fn to_chat_content(content: FixtureContent) -> ChatContent {
FixtureContentPart::ImageUrl { image_url } => {
ChatContentPart::image_url(image_url)
}
FixtureContentPart::InputAudio { data } => ChatContentPart::InputAudio {
data,
format: None,
uuid: None,
},
FixtureContentPart::AudioUrl { audio_url } => ChatContentPart::AudioUrl {
audio_url,
uuid: None,
},
})
.collect(),
),
+16 -2
View File
@@ -263,19 +263,32 @@ impl RoundtripCase {
sort_json_keys: false,
}
}
/// Inkling typed content blocks with native token-id rendering.
fn inkling() -> Self {
Self {
model_id: "thinkingmachines/Inkling",
assistant_stop_suffix: "",
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
thinking_behavior: ThinkingBehavior::Always { value: true },
json_fmt: compact_json_fmt(),
sort_json_keys: true,
}
}
}
macro_rules! roundtrip_tests {
($($case:ident => [$($(#[$fixture_attr:meta])* $fixture:ident),* $(,)?]),+ $(,)?) => {
($($case:ident => $(#[$case_attr:meta])* [$($fixture:ident),* $(,)?]),+ $(,)?) => {
paste::paste! {
$(
#[tokio::test]
#[file_serial([<hf_ $case>])]
$(#[$case_attr])*
async fn [<roundtrip_ $case>]() -> Result<()> {
let case = RoundtripCase::$case();
let backends = load_roundtrip_backends(&case).await?;
$(
$(#[$fixture_attr])*
[<run_roundtrip_ $fixture>](&case, &backends).await?;
)*
Ok(())
@@ -300,6 +313,7 @@ roundtrip_tests! {
gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call
kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history
gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call
inkling => [reasoning_and_content, tool_call_mix],
}
/// Run the fixed reasoning+content fixture for one model/parser case.