[Model] Add Inkling model support [1/N] (#48799)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Bugen Zhao
Giancarlo Delfin
Isotr0py
Isotr0py
Jee Jee Li
Roger Wang
Yifan Qiao
Claude Fable 5
OpenAI Codex
parent
8bfd683901
commit
6570c9800c
@@ -20,6 +20,7 @@ use crate::output::{
|
||||
use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
|
||||
use crate::renderer::{
|
||||
DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer,
|
||||
InklingChatRenderer,
|
||||
};
|
||||
use crate::request::ChatRequest;
|
||||
use crate::{DynChatOutputProcessor, RendererSelection};
|
||||
@@ -71,6 +72,7 @@ impl HfChatBackend {
|
||||
RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()),
|
||||
RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
|
||||
RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
|
||||
RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
|
||||
};
|
||||
|
||||
info!(
|
||||
|
||||
@@ -33,7 +33,7 @@ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
|
||||
pub use renderer::hf::ChatTemplateContentFormatOption;
|
||||
pub use renderer::{
|
||||
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
|
||||
HarmonyChatRenderer, RenderedPrompt, RendererSelection,
|
||||
HarmonyChatRenderer, InklingChatRenderer, RenderedPrompt, RendererSelection,
|
||||
};
|
||||
pub use request::{
|
||||
ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
|
||||
@@ -294,7 +294,7 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -305,6 +305,6 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,6 +79,38 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
const AUDIO_PAD_ID: u32 = 151_676;
|
||||
const INKLING_AUDIO_MARKER_ID: u32 = 200_020;
|
||||
const INKLING_AUDIO_EMBED_ID: i32 = 200_053;
|
||||
|
||||
fn inkling_info(decoder_dmodel: serde_json::Value) -> MultimodalModelInfo {
|
||||
let config = serde_json::json!({
|
||||
"model_type": "inkling_mm_model",
|
||||
"audio_config": {
|
||||
"decoder_dmodel": decoder_dmodel,
|
||||
"n_mel_bins": 80,
|
||||
"mel_vocab_size": 16,
|
||||
"dmel_min_value": -7.0,
|
||||
"dmel_max_value": 2.0
|
||||
}
|
||||
});
|
||||
let tokenizer = TestTokenizer::new()
|
||||
.with_regular_token("<|content_image|>", 200_005)
|
||||
.with_regular_token("<|content_audio_input|>", INKLING_AUDIO_MARKER_ID);
|
||||
let context = MultimodalModelContext {
|
||||
model_id: "inkling-test".to_string(),
|
||||
model_type: Some("inkling_mm_model".to_string()),
|
||||
config,
|
||||
tokenizer: TokenizerResolver(Arc::new(tokenizer)),
|
||||
};
|
||||
|
||||
MultimodalModelInfo::from_loaded(
|
||||
context,
|
||||
PreProcessorConfig::default(),
|
||||
PreProcessorConfig::default(),
|
||||
)
|
||||
.unwrap()
|
||||
.expect("Inkling multimodal support")
|
||||
}
|
||||
|
||||
fn qwen3_asr_info() -> MultimodalModelInfo {
|
||||
let context = MultimodalModelContext {
|
||||
@@ -120,6 +152,40 @@ mod tests {
|
||||
bytes
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_inkling_audio_from_model_spec() {
|
||||
let info = inkling_info(serde_json::json!(1024));
|
||||
let support = info.audio.as_ref().expect("audio support");
|
||||
|
||||
assert_eq!(
|
||||
info.placeholder_token(Modality::Audio),
|
||||
Some("<|content_audio_input|>")
|
||||
);
|
||||
assert_eq!(support.placeholder.marker_token_id, INKLING_AUDIO_MARKER_ID);
|
||||
assert_eq!(
|
||||
support.placeholder.embed_token_id,
|
||||
INKLING_AUDIO_EMBED_ID as u32
|
||||
);
|
||||
assert_eq!(support.spec.primary_key(), AUDIO_PRIMARY_KEY);
|
||||
assert!(matches!(
|
||||
&support.spec.field_layouts.encoder_input,
|
||||
llm_multimodal::FieldLayout::Flat { sizes_key }
|
||||
if sizes_key == "num_audio_tokens"
|
||||
));
|
||||
assert!(matches!(
|
||||
support.spec.field_layouts.model_specific.get("num_audio_tokens"),
|
||||
Some(llm_multimodal::FieldLayout::Batched)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_decoder_config_gates_audio_capability() {
|
||||
let info = inkling_info(serde_json::Value::Null);
|
||||
|
||||
assert!(info.audio.is_none());
|
||||
assert_eq!(info.placeholder_token(Modality::Audio), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_qwen_audio_from_model_spec() {
|
||||
let info = qwen3_asr_info();
|
||||
@@ -210,4 +276,50 @@ mod tests {
|
||||
if tensor.dtype == "int64" && tensor.shape.is_empty()
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inkling_tracker_and_processor_use_standard_audio_key() {
|
||||
let info = inkling_info(serde_json::json!(1024));
|
||||
let wav = wav_i16_mono(16_000, &[0; 1_600]);
|
||||
let expected_hash = llm_multimodal::hasher::hash_audio(&wav);
|
||||
let fetched = info
|
||||
.fetch_media(vec![MediaContentPart::AudioData {
|
||||
data: wav,
|
||||
mime_type: Some("audio/wav".to_string()),
|
||||
uuid: Some("audio-1".to_string()),
|
||||
}])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let prepared = info.prepare_audios(fetched.audios, fetched.audio_uuids).await.unwrap();
|
||||
|
||||
assert_eq!(prepared.replacements.len(), 1);
|
||||
assert_eq!(
|
||||
prepared.replacements[0].tokens[0],
|
||||
INKLING_AUDIO_MARKER_ID as i32
|
||||
);
|
||||
assert!(
|
||||
prepared.replacements[0].tokens[1..]
|
||||
.iter()
|
||||
.all(|token| *token == INKLING_AUDIO_EMBED_ID)
|
||||
);
|
||||
let item = &prepared.items[0];
|
||||
assert_eq!(item.hash, expected_hash);
|
||||
assert_eq!(item.uuid.as_deref(), Some("audio-1"));
|
||||
|
||||
let features = &item.data[AUDIO_PRIMARY_KEY];
|
||||
assert!(matches!(&features.field, MmField::Flat(_)));
|
||||
assert!(matches!(
|
||||
features.data.as_ref(),
|
||||
Some(MmKwargValue::Tensor(tensor))
|
||||
if tensor.dtype == "float32" && tensor.shape.get(1) == Some(&80)
|
||||
));
|
||||
let count = &item.data["num_audio_tokens"];
|
||||
assert!(matches!(&count.field, MmField::Batched(_)));
|
||||
assert!(matches!(
|
||||
count.data.as_ref(),
|
||||
Some(MmKwargValue::Tensor(tensor))
|
||||
if tensor.dtype == "int64" && tensor.shape.is_empty()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ pub mod names {
|
||||
pub const DEEPSEEK_V3: &str = "deepseek_v3";
|
||||
pub const DEEPSEEK_V4: &str = "deepseek_v4";
|
||||
pub const GEMMA4: &str = "gemma4";
|
||||
pub const INKLING: &str = "inkling";
|
||||
pub const GLM45: &str = "glm45";
|
||||
pub const KIMI: &str = "kimi";
|
||||
pub const KIMI_K2: &str = "kimi_k2";
|
||||
@@ -63,6 +64,7 @@ impl ReasoningParserFactory {
|
||||
.register_parser::<DeepSeekV3ReasoningParser>(names::DEEPSEEK_V3)
|
||||
.register_parser::<DeepSeekV4ReasoningParser>(names::DEEPSEEK_V4)
|
||||
.register_unified_dummy(names::GEMMA4)
|
||||
.register_unified_dummy(names::INKLING)
|
||||
.register_parser::<Glm45ReasoningParser>(names::GLM45)
|
||||
.register_parser::<KimiReasoningParser>(names::KIMI)
|
||||
.register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
|
||||
|
||||
@@ -25,6 +25,7 @@ pub mod names {
|
||||
pub const GLM45: &str = "glm45";
|
||||
pub const GLM47: &str = "glm47";
|
||||
pub const GEMMA4: &str = "gemma4";
|
||||
pub const INKLING: &str = "inkling";
|
||||
pub const GRANITE4: &str = "granite4";
|
||||
pub const HERMES: &str = "hermes";
|
||||
pub const HY_V3: &str = "hy_v3";
|
||||
@@ -70,6 +71,7 @@ impl ToolParserFactory {
|
||||
.register_parser::<Glm45MoeToolParser>(names::GLM45)
|
||||
.register_parser::<Glm47MoeToolParser>(names::GLM47)
|
||||
.register_unified_dummy(names::GEMMA4)
|
||||
.register_unified_dummy(names::INKLING)
|
||||
.register_parser::<Granite4ToolParser>(names::GRANITE4)
|
||||
.register_parser::<HermesToolParser>(names::HERMES)
|
||||
.register_parser::<HyV3ToolParser>(names::HY_V3)
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub use vllm_parser::unified::{Gemma4UnifiedParser, UnifiedParser};
|
||||
pub use vllm_parser::unified::{Gemma4UnifiedParser, InklingUnifiedParser, UnifiedParser};
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use crate::parser::ParserFactory;
|
||||
@@ -14,6 +14,7 @@ use crate::request::ChatTool;
|
||||
/// Canonical public names for registered unified parsers.
|
||||
pub mod names {
|
||||
pub const GEMMA4: &str = "gemma4";
|
||||
pub const INKLING: &str = "inkling";
|
||||
}
|
||||
|
||||
/// Constructor signature for one registered unified parser implementation.
|
||||
@@ -37,10 +38,12 @@ impl UnifiedParserFactory {
|
||||
let mut factory = Self::default();
|
||||
|
||||
factory.register_parser::<Gemma4UnifiedParser>(names::GEMMA4);
|
||||
factory.register_parser::<InklingUnifiedParser>(names::INKLING);
|
||||
|
||||
factory
|
||||
.register_pattern("gemma-4", names::GEMMA4)
|
||||
.register_pattern("gemma4", names::GEMMA4);
|
||||
.register_pattern("gemma4", names::GEMMA4)
|
||||
.register_pattern("inkling", names::INKLING);
|
||||
|
||||
factory
|
||||
}
|
||||
@@ -88,6 +91,13 @@ mod tests {
|
||||
.with_regular_token("<channel|>", 257)
|
||||
}
|
||||
|
||||
fn inkling_tokenizer() -> TestTokenizer {
|
||||
TestTokenizer::new()
|
||||
.with_regular_token("<|message_model|>", 200001)
|
||||
.with_regular_token("<|content_text|>", 200004)
|
||||
.with_regular_token("<|content_thinking|>", 200008)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_registers_gemma4() {
|
||||
let factory = UnifiedParserFactory::new();
|
||||
@@ -99,4 +109,16 @@ mod tests {
|
||||
);
|
||||
factory.create(names::GEMMA4, &[], Arc::new(tokenizer())).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_registers_inkling() {
|
||||
let factory = UnifiedParserFactory::new();
|
||||
|
||||
assert!(factory.contains(names::INKLING));
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("thinkingmachines/Inkling"),
|
||||
Some(names::INKLING)
|
||||
);
|
||||
factory.create(names::INKLING, &[], Arc::new(inkling_tokenizer())).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "transcribe"
|
||||
},
|
||||
{
|
||||
"type": "input_audio",
|
||||
"data": ""
|
||||
},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": "data:audio/wav;base64,"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>transcribe<|end_message|><|message_user|><|content_audio_input|><|audio_end|><|end_message|><|message_user|><|content_audio_input|><|audio_end|><|end_message|><|message_model|>
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "look"
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": "data:image/png;base64,"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>look<|end_message|><|message_user|><|content_image|><|end_message|><|message_model|>
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather information",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"city"
|
||||
],
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "developer",
|
||||
"content": "rules",
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "local_tool",
|
||||
"parameters": {
|
||||
"z": 1,
|
||||
"a": {
|
||||
"b": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "hi"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<|message_system|>tool_declare<|content_xml|>[{"description":"Get weather information","name":"get_weather","parameters":{"properties":{"city":{"type":"string"}},"required":["city"],"type":"object"},"type":"function"},{"description":"","name":"local_tool","parameters":{"a":{"b":2},"z":1},"type":"function"}]<|end_message|><|message_system|><|content_text|>rules<|end_message|><|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>hi<|end_message|><|message_model|>
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"add_generation_prompt": false,
|
||||
"messages": [
|
||||
{
|
||||
"role": "assistant",
|
||||
"reasoning_content": "think",
|
||||
"content": "answer",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": "{\"city\":\"SF\"}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": "sunny"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_model|><|content_thinking|>think<|end_message|><|message_model|><|content_text|>answer<|end_message|><|message_model|>get_weather<|content_invoke_tool_json|>{"name":"get_weather","args":{"city":"SF"}}<|end_message|><|content_model_end_sampling|><|message_tool|>get_weather<|content_text|>sunny<|end_message|>
|
||||
@@ -0,0 +1,442 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde_json::{Map, Value, json};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use vllm_text::Prompt;
|
||||
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
|
||||
|
||||
use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool};
|
||||
use crate::{AssistantContentBlock, AssistantToolCall};
|
||||
|
||||
const MESSAGE_USER: &str = "<|message_user|>";
|
||||
const MESSAGE_MODEL: &str = "<|message_model|>";
|
||||
const MESSAGE_SYSTEM: &str = "<|message_system|>";
|
||||
const MESSAGE_TOOL: &str = "<|message_tool|>";
|
||||
const CONTENT_TEXT: &str = "<|content_text|>";
|
||||
const CONTENT_IMAGE: &str = "<|content_image|>";
|
||||
const CONTENT_MODEL_END_SAMPLING: &str = "<|content_model_end_sampling|>";
|
||||
const CONTENT_AUDIO_INPUT: &str = "<|content_audio_input|>";
|
||||
const CONTENT_THINKING: &str = "<|content_thinking|>";
|
||||
// Inkling renderer semantics use this slot for structured payloads such as tool
|
||||
// declarations.
|
||||
const CONTENT_XML: &str = "<|content_xml|>";
|
||||
const CONTENT_INVOKE_TOOL_JSON: &str = "<|content_invoke_tool_json|>";
|
||||
const END_MESSAGE: &str = "<|end_message|>";
|
||||
const AUDIO_END: &str = "<|audio_end|>";
|
||||
const MAX_REASONING_EFFORT: f64 = 0.99;
|
||||
|
||||
/// Native Inkling renderer that emits token IDs directly.
|
||||
#[derive(Clone)]
|
||||
pub struct InklingChatRenderer {
|
||||
tokenizer: DynTokenizer,
|
||||
special: InklingSpecialTokenIds,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct InklingSpecialTokenIds {
|
||||
message_user: u32,
|
||||
message_model: u32,
|
||||
message_system: u32,
|
||||
message_tool: u32,
|
||||
content_text: u32,
|
||||
content_image: u32,
|
||||
content_model_end_sampling: u32,
|
||||
content_audio_input: u32,
|
||||
content_thinking: u32,
|
||||
content_xml: u32,
|
||||
content_invoke_tool_json: u32,
|
||||
end_message: u32,
|
||||
audio_end: u32,
|
||||
}
|
||||
|
||||
impl InklingSpecialTokenIds {
|
||||
fn resolve(tokenizer: &dyn Tokenizer) -> Result<Self> {
|
||||
Ok(Self {
|
||||
message_user: resolve_special_token(tokenizer, MESSAGE_USER)?,
|
||||
message_model: resolve_special_token(tokenizer, MESSAGE_MODEL)?,
|
||||
message_system: resolve_special_token(tokenizer, MESSAGE_SYSTEM)?,
|
||||
message_tool: resolve_special_token(tokenizer, MESSAGE_TOOL)?,
|
||||
content_text: resolve_special_token(tokenizer, CONTENT_TEXT)?,
|
||||
content_image: resolve_special_token(tokenizer, CONTENT_IMAGE)?,
|
||||
content_model_end_sampling: resolve_special_token(
|
||||
tokenizer,
|
||||
CONTENT_MODEL_END_SAMPLING,
|
||||
)?,
|
||||
content_audio_input: resolve_special_token(tokenizer, CONTENT_AUDIO_INPUT)?,
|
||||
content_thinking: resolve_special_token(tokenizer, CONTENT_THINKING)?,
|
||||
content_xml: resolve_special_token(tokenizer, CONTENT_XML)?,
|
||||
content_invoke_tool_json: resolve_special_token(tokenizer, CONTENT_INVOKE_TOOL_JSON)?,
|
||||
end_message: resolve_special_token(tokenizer, END_MESSAGE)?,
|
||||
audio_end: resolve_special_token(tokenizer, AUDIO_END)?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl InklingChatRenderer {
|
||||
pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
|
||||
let special = InklingSpecialTokenIds::resolve(tokenizer.as_ref())?;
|
||||
Ok(Self { tokenizer, special })
|
||||
}
|
||||
|
||||
fn write_text_tokens(&self, out: &mut Vec<u32>, text: &str) -> Result<()> {
|
||||
out.extend(self.tokenizer.encode(text, false)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_message_start(
|
||||
&self,
|
||||
out: &mut Vec<u32>,
|
||||
role_token_id: u32,
|
||||
author_name: Option<&str>,
|
||||
) -> Result<()> {
|
||||
out.push(role_token_id);
|
||||
if let Some(author_name) = author_name
|
||||
&& !author_name.is_empty()
|
||||
{
|
||||
self.write_text_tokens(out, author_name)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_text_block(
|
||||
&self,
|
||||
out: &mut Vec<u32>,
|
||||
role_token_id: u32,
|
||||
author_name: Option<&str>,
|
||||
text: &str,
|
||||
) -> Result<()> {
|
||||
self.write_message_start(out, role_token_id, author_name)?;
|
||||
out.push(self.special.content_text);
|
||||
self.write_text_tokens(out, text)?;
|
||||
out.push(self.special.end_message);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write one image block holding only the `<|content_image|>` marker.
|
||||
/// Multimodal preprocessing later expands the marker into per-patch
|
||||
/// image placeholder tokens once the patch count is known, mirroring the
|
||||
/// Python `InklingMultiModalProcessor` marker-anchored prompt updates.
|
||||
fn write_image_block(&self, out: &mut Vec<u32>, role_token_id: u32) {
|
||||
out.push(role_token_id);
|
||||
out.push(self.special.content_image);
|
||||
out.push(self.special.end_message);
|
||||
}
|
||||
|
||||
/// Write one audio block holding the `<|content_audio_input|>` marker
|
||||
/// followed by the `<|audio_end|>` terminator. Multimodal preprocessing
|
||||
/// later expands the marker into per-frame audio placeholder tokens
|
||||
/// (landing before `<|audio_end|>`) once the clip length is known.
|
||||
fn write_audio_block(&self, out: &mut Vec<u32>, role_token_id: u32) {
|
||||
out.push(role_token_id);
|
||||
out.push(self.special.content_audio_input);
|
||||
out.push(self.special.audio_end);
|
||||
out.push(self.special.end_message);
|
||||
}
|
||||
|
||||
fn write_reasoning_block(&self, out: &mut Vec<u32>, text: &str) -> Result<()> {
|
||||
if text.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
out.push(self.special.message_model);
|
||||
out.push(self.special.content_thinking);
|
||||
self.write_text_tokens(out, text)?;
|
||||
out.push(self.special.end_message);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_reasoning_effort(&self, out: &mut Vec<u32>, effort: f64) -> Result<()> {
|
||||
if !(0.0..=MAX_REASONING_EFFORT).contains(&effort) {
|
||||
return Err(Error::ChatTemplate(format!(
|
||||
"Inkling reasoning_effort must be in [0.0, 0.99], got {effort}"
|
||||
)));
|
||||
}
|
||||
let formatted = format!("{effort:.2}");
|
||||
let effort = formatted.trim_end_matches('0').trim_end_matches('.');
|
||||
let effort = if matches!(effort, "0" | "-0") {
|
||||
"0.0"
|
||||
} else {
|
||||
effort
|
||||
};
|
||||
self.write_text_block(
|
||||
out,
|
||||
self.special.message_system,
|
||||
None,
|
||||
&format!("Thinking effort level: {effort}"),
|
||||
)
|
||||
}
|
||||
|
||||
fn write_tool_declarations(&self, out: &mut Vec<u32>, tools: &[&ChatTool]) -> Result<()> {
|
||||
if tools.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut specs = Vec::with_capacity(tools.len());
|
||||
for tool in tools {
|
||||
specs.push(json!({
|
||||
"description": tool.description.as_deref().unwrap_or(""),
|
||||
"name": tool.name,
|
||||
"parameters": sort_json(&tool.parameters),
|
||||
"type": "function",
|
||||
}));
|
||||
}
|
||||
let payload = compact_json(&sort_json(&Value::Array(specs)))?;
|
||||
|
||||
self.write_message_start(out, self.special.message_system, Some("tool_declare"))?;
|
||||
out.push(self.special.content_xml);
|
||||
self.write_text_tokens(out, &payload)?;
|
||||
out.push(self.special.end_message);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_chat_content(
|
||||
&self,
|
||||
out: &mut Vec<u32>,
|
||||
role_token_id: u32,
|
||||
content: &ChatContent,
|
||||
) -> Result<()> {
|
||||
match content {
|
||||
ChatContent::Text(text) => {
|
||||
if !text.is_empty() {
|
||||
self.write_text_block(out, role_token_id, None, text)?;
|
||||
}
|
||||
}
|
||||
ChatContent::Parts(parts) => {
|
||||
for part in parts {
|
||||
match part {
|
||||
ChatContentPart::Text { text } => {
|
||||
self.write_text_block(out, role_token_id, None, text)?;
|
||||
}
|
||||
ChatContentPart::ImageUrl { .. } => {
|
||||
self.write_image_block(out, role_token_id);
|
||||
}
|
||||
ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => {
|
||||
self.write_audio_block(out, role_token_id);
|
||||
}
|
||||
// Inkling has no video modality.
|
||||
ChatContentPart::VideoUrl { .. } => {
|
||||
return Err(Error::UnsupportedMultimodalContent("video_url"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_assistant_tool_call(
|
||||
&self,
|
||||
out: &mut Vec<u32>,
|
||||
tool_call: &AssistantToolCall,
|
||||
) -> Result<()> {
|
||||
let payload = tool_call_json(tool_call)?;
|
||||
|
||||
self.write_message_start(out, self.special.message_model, Some(&tool_call.name))?;
|
||||
out.push(self.special.content_invoke_tool_json);
|
||||
self.write_text_tokens(out, &payload)?;
|
||||
out.push(self.special.end_message);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_assistant_content(
|
||||
&self,
|
||||
out: &mut Vec<u32>,
|
||||
content: &[AssistantContentBlock],
|
||||
tool_call_id_to_name: &mut HashMap<String, String>,
|
||||
) -> Result<()> {
|
||||
for block in content {
|
||||
match block {
|
||||
AssistantContentBlock::Reasoning { text } => {
|
||||
self.write_reasoning_block(out, text)?;
|
||||
}
|
||||
AssistantContentBlock::Text { text } => {
|
||||
if !text.is_empty() {
|
||||
self.write_text_block(out, self.special.message_model, None, text)?;
|
||||
}
|
||||
}
|
||||
AssistantContentBlock::ToolCall(tool_call) => {
|
||||
if !tool_call.id.is_empty() {
|
||||
tool_call_id_to_name.insert(tool_call.id.clone(), tool_call.name.clone());
|
||||
}
|
||||
self.write_assistant_tool_call(out, tool_call)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push(self.special.content_model_end_sampling);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_tool_response(
|
||||
&self,
|
||||
out: &mut Vec<u32>,
|
||||
content: &ChatContent,
|
||||
tool_call_id: &str,
|
||||
tool_call_id_to_name: &HashMap<String, String>,
|
||||
) -> Result<()> {
|
||||
let text = content.try_flatten_to_text()?;
|
||||
let tool_name = tool_call_id_to_name.get(tool_call_id).map(String::as_str).unwrap_or("");
|
||||
self.write_text_block(out, self.special.message_tool, Some(tool_name), &text)
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRenderer for InklingChatRenderer {
|
||||
fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
|
||||
request.validate()?;
|
||||
if request.chat_options.continue_final_message() {
|
||||
return Err(Error::ChatTemplate(
|
||||
"Inkling renderer does not support continue_final_message".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut out = Vec::new();
|
||||
let mut tool_call_id_to_name = HashMap::new();
|
||||
let effective_template_kwargs = request_template_kwargs(request);
|
||||
let tools = rendered_tools(request);
|
||||
self.write_tool_declarations(&mut out, &tools)?;
|
||||
let mut reasoning_effort =
|
||||
resolve_reasoning_effort(effective_template_kwargs.get("reasoning_effort"));
|
||||
|
||||
for message in &request.messages {
|
||||
if !matches!(
|
||||
message,
|
||||
ChatMessage::System { .. } | ChatMessage::Developer { .. }
|
||||
) && let Some(effort) = reasoning_effort.take()
|
||||
{
|
||||
self.write_reasoning_effort(&mut out, effort)?;
|
||||
}
|
||||
|
||||
match message {
|
||||
ChatMessage::System { content } => {
|
||||
self.write_chat_content(&mut out, self.special.message_system, content)?;
|
||||
}
|
||||
ChatMessage::Developer { content, .. } => {
|
||||
self.write_chat_content(&mut out, self.special.message_system, content)?;
|
||||
}
|
||||
ChatMessage::User { content } => {
|
||||
self.write_chat_content(&mut out, self.special.message_user, content)?;
|
||||
}
|
||||
ChatMessage::Assistant { content } => {
|
||||
self.write_assistant_content(&mut out, content, &mut tool_call_id_to_name)?;
|
||||
}
|
||||
ChatMessage::ToolResponse {
|
||||
content,
|
||||
tool_call_id,
|
||||
} => {
|
||||
self.write_tool_response(
|
||||
&mut out,
|
||||
content,
|
||||
tool_call_id,
|
||||
&tool_call_id_to_name,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(effort) = reasoning_effort {
|
||||
self.write_reasoning_effort(&mut out, effort)?;
|
||||
}
|
||||
|
||||
if request.chat_options.add_generation_prompt() {
|
||||
out.push(self.special.message_model);
|
||||
}
|
||||
|
||||
Ok(RenderedPrompt {
|
||||
prompt: Prompt::TokenIds(out),
|
||||
effective_template_kwargs,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_reasoning_effort(value: Option<&Value>) -> Option<f64> {
|
||||
let Some(value) = value else {
|
||||
return Some(0.9);
|
||||
};
|
||||
match value {
|
||||
Value::String(name) => match name.as_str() {
|
||||
"none" => Some(0.0),
|
||||
"minimal" => Some(0.1),
|
||||
"low" => Some(0.2),
|
||||
"medium" => Some(0.7),
|
||||
"high" => Some(0.9),
|
||||
"xhigh" | "max" => Some(0.99),
|
||||
_ => None,
|
||||
},
|
||||
Value::Number(number) => number.as_f64(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_special_token(tokenizer: &dyn Tokenizer, token: &str) -> Result<u32> {
|
||||
tokenizer.token_to_id(token).ok_or_else(|| {
|
||||
Error::ChatTemplate(format!(
|
||||
"Inkling tokenizer is missing special token `{token}`"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn rendered_tools(request: &ChatRequest) -> Vec<&ChatTool> {
|
||||
if !request.tool_parsing_enabled() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut tools = Vec::with_capacity(request.tools.len());
|
||||
tools.extend(request.tools.iter());
|
||||
for message in &request.messages {
|
||||
if let ChatMessage::Developer {
|
||||
tools: Some(local_tools),
|
||||
..
|
||||
} = message
|
||||
{
|
||||
tools.extend(local_tools.iter());
|
||||
}
|
||||
}
|
||||
tools
|
||||
}
|
||||
|
||||
fn tool_call_json(tool_call: &AssistantToolCall) -> Result<String> {
|
||||
let name_json = serde_json::to_string(&tool_call.name)
|
||||
.map_err(|error| Error::ChatTemplate(error.as_report().to_string()))?;
|
||||
let arguments = if tool_call.arguments.trim().is_empty() {
|
||||
Value::Object(Map::new())
|
||||
} else {
|
||||
serde_json::from_str(&tool_call.arguments).map_err(|error| {
|
||||
Error::ChatTemplate(format!(
|
||||
"Inkling tool call arguments must decode to a JSON object: {error}"
|
||||
))
|
||||
})?
|
||||
};
|
||||
let Value::Object(_) = arguments else {
|
||||
return Err(Error::ChatTemplate(
|
||||
"Inkling tool call arguments must decode to a JSON object".to_string(),
|
||||
));
|
||||
};
|
||||
let args_json = compact_json(&sort_json(&arguments))?;
|
||||
Ok(format!("{{\"name\":{name_json},\"args\":{args_json}}}"))
|
||||
}
|
||||
|
||||
fn compact_json(value: &Value) -> Result<String> {
|
||||
serde_json::to_string(value).map_err(|error| Error::ChatTemplate(error.as_report().to_string()))
|
||||
}
|
||||
|
||||
fn sort_json(value: &Value) -> Value {
|
||||
match value {
|
||||
Value::Array(items) => Value::Array(items.iter().map(sort_json).collect()),
|
||||
Value::Object(map) => {
|
||||
let mut sorted = Map::new();
|
||||
let mut keys = map.keys().collect::<Vec<_>>();
|
||||
keys.sort();
|
||||
for key in keys {
|
||||
sorted.insert(key.clone(), sort_json(&map[key]));
|
||||
}
|
||||
Value::Object(sorted)
|
||||
}
|
||||
_ => value.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,375 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use expect_test::{ExpectFile, expect_file};
|
||||
use serde_json::json;
|
||||
use thiserror_ext::AsReport;
|
||||
use vllm_text::tokenizer::Tokenizer;
|
||||
|
||||
use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
|
||||
|
||||
use super::{
|
||||
AUDIO_END, CONTENT_AUDIO_INPUT, CONTENT_IMAGE, CONTENT_INVOKE_TOOL_JSON, CONTENT_TEXT,
|
||||
CONTENT_THINKING, CONTENT_XML, END_MESSAGE, InklingChatRenderer, MESSAGE_MODEL, MESSAGE_SYSTEM,
|
||||
MESSAGE_TOOL, MESSAGE_USER,
|
||||
};
|
||||
use crate::event::{AssistantContentBlock, AssistantToolCall};
|
||||
use crate::request::{ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort};
|
||||
use crate::{ChatRenderer, Error};
|
||||
|
||||
struct FixtureTokenizer;
|
||||
|
||||
impl Tokenizer for FixtureTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.bytes().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_text::tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<|message_user|>" => Some(200000),
|
||||
"<|message_model|>" => Some(200001),
|
||||
"<|message_system|>" => Some(200002),
|
||||
"<|message_tool|>" => Some(200003),
|
||||
"<|content_text|>" => Some(200004),
|
||||
"<|content_image|>" => Some(200005),
|
||||
"<|content_model_end_sampling|>" => Some(200006),
|
||||
"<|content_thinking|>" => Some(200008),
|
||||
"<|end_message|>" => Some(200010),
|
||||
"<|content_audio_input|>" => Some(200020),
|
||||
CONTENT_XML => Some(200024),
|
||||
"<|audio_end|>" => Some(200043),
|
||||
"<|content_invoke_tool_json|>" => Some(200049),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
let token = match id {
|
||||
200000 => "<|message_user|>",
|
||||
200001 => "<|message_model|>",
|
||||
200002 => "<|message_system|>",
|
||||
200003 => "<|message_tool|>",
|
||||
200004 => "<|content_text|>",
|
||||
200005 => "<|content_image|>",
|
||||
200006 => "<|content_model_end_sampling|>",
|
||||
200008 => "<|content_thinking|>",
|
||||
200010 => "<|end_message|>",
|
||||
200020 => "<|content_audio_input|>",
|
||||
200024 => CONTENT_XML,
|
||||
200043 => "<|audio_end|>",
|
||||
200049 => "<|content_invoke_tool_json|>",
|
||||
_ => return None,
|
||||
};
|
||||
Some(token.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn renderer() -> InklingChatRenderer {
|
||||
InklingChatRenderer::new(Arc::new(FixtureTokenizer)).unwrap()
|
||||
}
|
||||
|
||||
fn render_token_ids(request: &ChatRequest) -> Vec<u32> {
|
||||
renderer()
|
||||
.render(request)
|
||||
.unwrap()
|
||||
.prompt
|
||||
.into_token_ids()
|
||||
.expect("Inkling renderer returns token IDs")
|
||||
}
|
||||
|
||||
fn fixture_request(name: &str) -> ChatRequest {
|
||||
fixture_chat_request(&fixture_path(name), inkling_fixture_options())
|
||||
}
|
||||
|
||||
fn inkling_fixture_options() -> FixtureRequestOptions {
|
||||
FixtureRequestOptions {
|
||||
enable_thinking: false,
|
||||
no_generation_prompt_when_last_assistant: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn fixture_path(name: &str) -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("src/renderer/inkling")
|
||||
.join("fixtures")
|
||||
.join(name)
|
||||
}
|
||||
|
||||
fn assert_fixture(input_name: &str, expected: ExpectFile) {
|
||||
let request = fixture_request(input_name);
|
||||
let rendered = format!("{}\n", render_symbolic_tokens(&render_token_ids(&request)));
|
||||
|
||||
expected.assert_eq(&rendered);
|
||||
}
|
||||
|
||||
fn render_symbolic_tokens(token_ids: &[u32]) -> String {
|
||||
let mut out = String::new();
|
||||
let mut text_bytes = Vec::new();
|
||||
for token_id in token_ids {
|
||||
if let Some(marker) = symbolic_tokens()
|
||||
.iter()
|
||||
.find_map(|(marker, marker_id)| (token_id == marker_id).then_some(*marker))
|
||||
{
|
||||
flush_text_bytes(&mut out, &mut text_bytes);
|
||||
out.push_str(marker);
|
||||
continue;
|
||||
}
|
||||
|
||||
let byte = u8::try_from(*token_id)
|
||||
.unwrap_or_else(|_| panic!("unexpected non-byte fixture text token {token_id}"));
|
||||
text_bytes.push(byte);
|
||||
}
|
||||
flush_text_bytes(&mut out, &mut text_bytes);
|
||||
out
|
||||
}
|
||||
|
||||
fn flush_text_bytes(out: &mut String, text_bytes: &mut Vec<u8>) {
|
||||
if text_bytes.is_empty() {
|
||||
return;
|
||||
}
|
||||
out.push_str(std::str::from_utf8(text_bytes).unwrap());
|
||||
text_bytes.clear();
|
||||
}
|
||||
|
||||
fn symbolic_tokens() -> [(&'static str, u32); 13] {
|
||||
[
|
||||
(MESSAGE_USER, 200000),
|
||||
(MESSAGE_MODEL, 200001),
|
||||
(MESSAGE_SYSTEM, 200002),
|
||||
(MESSAGE_TOOL, 200003),
|
||||
(CONTENT_TEXT, 200004),
|
||||
(CONTENT_IMAGE, 200005),
|
||||
(super::CONTENT_MODEL_END_SAMPLING, 200006),
|
||||
(CONTENT_THINKING, 200008),
|
||||
(END_MESSAGE, 200010),
|
||||
(CONTENT_AUDIO_INPUT, 200020),
|
||||
(CONTENT_XML, 200024),
|
||||
(AUDIO_END, 200043),
|
||||
(CONTENT_INVOKE_TOOL_JSON, 200049),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_text_and_image_as_inkling_tokens() {
|
||||
assert_fixture(
|
||||
"text_image_input.json",
|
||||
expect_file!["fixtures/text_image_output.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_audio_marker_only_blocks() {
|
||||
assert_fixture(
|
||||
"text_audio_input.json",
|
||||
expect_file!["fixtures/text_audio_output.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_reasoning_text_tool_call_and_tool_response() {
|
||||
assert_fixture(
|
||||
"tool_round_trip_input.json",
|
||||
expect_file!["fixtures/tool_round_trip_output.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_request_and_developer_tools_as_tool_declare() {
|
||||
assert_fixture(
|
||||
"tool_declare_input.json",
|
||||
expect_file!["fixtures/tool_declare_output.txt"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_named_reasoning_effort_after_tool_declarations() {
|
||||
for (effort, expected) in [
|
||||
(ReasoningEffort::None, "0.0"),
|
||||
(ReasoningEffort::Minimal, "0.1"),
|
||||
(ReasoningEffort::Low, "0.2"),
|
||||
(ReasoningEffort::Medium, "0.7"),
|
||||
(ReasoningEffort::High, "0.9"),
|
||||
(ReasoningEffort::XHigh, "0.99"),
|
||||
(ReasoningEffort::Max, "0.99"),
|
||||
] {
|
||||
let mut request = fixture_request("tool_declare_input.json");
|
||||
request.chat_options.reasoning_effort = Some(effort);
|
||||
|
||||
let rendered = render_symbolic_tokens(&render_token_ids(&request));
|
||||
let tool_end = rendered.find("<|end_message|>").unwrap();
|
||||
let effort_block = format!(
|
||||
"<|message_system|><|content_text|>Thinking effort level: \
|
||||
{expected}<|end_message|>"
|
||||
);
|
||||
let effort_start = rendered.find(&effort_block).unwrap();
|
||||
let system_end = rendered.find("rules<|end_message|>").unwrap();
|
||||
|
||||
assert!(effort_start > tool_end);
|
||||
assert!(effort_start > system_end);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emits_one_reasoning_effort_for_multi_turn_conversation() {
|
||||
let request = ChatRequest {
|
||||
messages: vec![
|
||||
ChatMessage::system("rules"),
|
||||
ChatMessage::user("user1"),
|
||||
ChatMessage::assistant_text("assistant1"),
|
||||
ChatMessage::user("user2"),
|
||||
],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
let rendered = render_symbolic_tokens(&render_token_ids(&request));
|
||||
let effort = "<|message_system|><|content_text|>Thinking effort level: 0.9\
|
||||
<|end_message|>";
|
||||
|
||||
assert_eq!(rendered.matches(effort).count(), 1);
|
||||
assert!(rendered.find("rules").unwrap() < rendered.find(effort).unwrap());
|
||||
assert!(rendered.find(effort).unwrap() < rendered.find("user1").unwrap());
|
||||
assert!(rendered.find("user1").unwrap() < rendered.find("assistant1").unwrap());
|
||||
assert!(rendered.find("assistant1").unwrap() < rendered.find("user2").unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalizes_numeric_zero_reasoning_effort() {
|
||||
for value in [0.0, -0.0] {
|
||||
let mut request = ChatRequest::for_test();
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("reasoning_effort".to_string(), json!(value));
|
||||
|
||||
assert!(
|
||||
render_symbolic_tokens(&render_token_ids(&request)).starts_with(
|
||||
"<|message_system|><|content_text|>Thinking effort level: 0.0\
|
||||
<|end_message|>"
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_reasoning_effort_to_high() {
|
||||
let request = ChatRequest::for_test();
|
||||
|
||||
assert!(
|
||||
render_symbolic_tokens(&render_token_ids(&request)).starts_with(
|
||||
"<|message_system|><|content_text|>Thinking effort level: 0.9\
|
||||
<|end_message|>"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_numeric_reasoning_effort_template_kwarg() {
|
||||
let mut request = ChatRequest::for_test();
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("reasoning_effort".to_string(), json!(0.8));
|
||||
|
||||
assert_eq!(
|
||||
render_symbolic_tokens(&render_token_ids(&request)),
|
||||
"<|message_system|><|content_text|>Thinking effort level: 0.8\
|
||||
<|end_message|><|message_user|><|content_text|>test\
|
||||
<|end_message|><|message_model|>"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_unsupported_reasoning_effort_values() {
|
||||
for value in [json!(true), json!("invalid"), json!(null)] {
|
||||
let mut request = ChatRequest::for_test();
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("reasoning_effort".to_string(), value);
|
||||
|
||||
assert_eq!(
|
||||
render_symbolic_tokens(&render_token_ids(&request)),
|
||||
"<|message_user|><|content_text|>test<|end_message|><|message_model|>"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_out_of_range_reasoning_effort() {
|
||||
for value in [0.990_000_1, 1.0, 1.5, -0.1] {
|
||||
let mut request = ChatRequest::for_test();
|
||||
request
|
||||
.chat_options
|
||||
.template_kwargs
|
||||
.insert("reasoning_effort".to_string(), json!(value));
|
||||
|
||||
let error = renderer().render(&request).unwrap_err();
|
||||
assert!(error.as_report().to_string().contains("must be in [0.0, 0.99]"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_continue_final_message() {
|
||||
let request = ChatRequest {
|
||||
messages: vec![ChatMessage::assistant_text("partial")],
|
||||
chat_options: crate::ChatOptions {
|
||||
generation_prompt_mode: GenerationPromptMode::ContinueFinalAssistant,
|
||||
..Default::default()
|
||||
},
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
let error = renderer().render(&request).unwrap_err();
|
||||
assert!(matches!(error, Error::ChatTemplate(_)));
|
||||
assert!(error.as_report().to_string().contains("continue_final_message"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renders_developer_messages_as_system() {
|
||||
let request = ChatRequest {
|
||||
messages: vec![ChatMessage::developer("rules", None)],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
render_symbolic_tokens(&render_token_ids(&request)),
|
||||
"<|message_system|><|content_text|>rules<|end_message|>\
|
||||
<|message_system|><|content_text|>Thinking effort level: 0.9\
|
||||
<|end_message|><|message_model|>"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_object_tool_call_arguments() {
|
||||
let request = ChatRequest {
|
||||
messages: vec![ChatMessage::assistant_blocks(vec![
|
||||
AssistantContentBlock::ToolCall(AssistantToolCall {
|
||||
id: "call_1".to_string(),
|
||||
name: "get_weather".to_string(),
|
||||
arguments: "[]".to_string(),
|
||||
}),
|
||||
])],
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
let error = renderer().render(&request).unwrap_err();
|
||||
assert!(error.as_report().to_string().contains("JSON object"));
|
||||
}
|
||||
@@ -14,6 +14,7 @@ pub mod deepseek_v32;
|
||||
pub mod deepseek_v4;
|
||||
pub mod harmony;
|
||||
pub mod hf;
|
||||
mod inkling;
|
||||
mod selection;
|
||||
#[cfg(test)]
|
||||
mod test_utils;
|
||||
@@ -21,6 +22,7 @@ mod test_utils;
|
||||
pub use deepseek_v4::DeepSeekV4ChatRenderer;
|
||||
pub use deepseek_v32::DeepSeekV32ChatRenderer;
|
||||
pub use harmony::HarmonyChatRenderer;
|
||||
pub use inkling::InklingChatRenderer;
|
||||
pub use selection::RendererSelection;
|
||||
|
||||
/// Rendered chat prompt submitted to the text backend.
|
||||
|
||||
@@ -24,6 +24,8 @@ pub enum RendererSelection {
|
||||
DeepSeekV4,
|
||||
/// Force the GPT-OSS Harmony renderer.
|
||||
Harmony,
|
||||
/// Force the Inkling native token renderer.
|
||||
Inkling,
|
||||
}
|
||||
|
||||
impl RendererSelection {
|
||||
@@ -33,6 +35,8 @@ impl RendererSelection {
|
||||
pub const GPT_OSS_MODEL_TYPE: &str = "gpt_oss";
|
||||
pub const HARMONY_LITERAL: &str = "harmony";
|
||||
pub const HF_LITERAL: &str = "hf";
|
||||
pub const INKLING_LITERAL: &str = "inkling";
|
||||
pub const INKLING_MODEL_TYPE: &str = "inkling_mm_model";
|
||||
|
||||
/// Resolve the renderer selection using the given model type string, if
|
||||
/// it's `Auto`.
|
||||
@@ -42,6 +46,7 @@ impl RendererSelection {
|
||||
Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32,
|
||||
Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
|
||||
Self::GPT_OSS_MODEL_TYPE => Self::Harmony,
|
||||
Self::INKLING_MODEL_TYPE => Self::Inkling,
|
||||
_ => Self::Hf,
|
||||
},
|
||||
selection => selection,
|
||||
@@ -63,6 +68,8 @@ impl FromStr for RendererSelection {
|
||||
Ok(Self::DeepSeekV4)
|
||||
} else if value.eq_ignore_ascii_case(Self::HARMONY_LITERAL) {
|
||||
Ok(Self::Harmony)
|
||||
} else if value.eq_ignore_ascii_case(Self::INKLING_LITERAL) {
|
||||
Ok(Self::Inkling)
|
||||
} else {
|
||||
Err(format!(
|
||||
"unknown renderer `{value}` (expected one of: {})",
|
||||
@@ -80,6 +87,7 @@ impl fmt::Display for RendererSelection {
|
||||
Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL),
|
||||
Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
|
||||
Self::Harmony => f.write_str(Self::HARMONY_LITERAL),
|
||||
Self::Inkling => f.write_str(Self::INKLING_LITERAL),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -106,7 +114,7 @@ mod tests {
|
||||
fn renderer_selection_expected_error_message() {
|
||||
let err = RendererSelection::from_str("unknown").unwrap_err();
|
||||
expect_test::expect![
|
||||
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)"
|
||||
"unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling)"
|
||||
]
|
||||
.assert_eq(&err);
|
||||
}
|
||||
|
||||
@@ -103,6 +103,8 @@ pub(crate) enum FixtureContent {
|
||||
pub(crate) enum FixtureContentPart {
|
||||
Text { text: String },
|
||||
ImageUrl { image_url: String },
|
||||
InputAudio { data: String },
|
||||
AudioUrl { audio_url: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -226,6 +228,15 @@ fn to_chat_content(content: FixtureContent) -> ChatContent {
|
||||
FixtureContentPart::ImageUrl { image_url } => {
|
||||
ChatContentPart::image_url(image_url)
|
||||
}
|
||||
FixtureContentPart::InputAudio { data } => ChatContentPart::InputAudio {
|
||||
data,
|
||||
format: None,
|
||||
uuid: None,
|
||||
},
|
||||
FixtureContentPart::AudioUrl { audio_url } => ChatContentPart::AudioUrl {
|
||||
audio_url,
|
||||
uuid: None,
|
||||
},
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
|
||||
@@ -263,19 +263,32 @@ impl RoundtripCase {
|
||||
sort_json_keys: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Inkling typed content blocks with native token-id rendering.
|
||||
fn inkling() -> Self {
|
||||
Self {
|
||||
model_id: "thinkingmachines/Inkling",
|
||||
assistant_stop_suffix: "",
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Always { value: true },
|
||||
json_fmt: compact_json_fmt(),
|
||||
sort_json_keys: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! roundtrip_tests {
|
||||
($($case:ident => [$($(#[$fixture_attr:meta])* $fixture:ident),* $(,)?]),+ $(,)?) => {
|
||||
($($case:ident => $(#[$case_attr:meta])* [$($fixture:ident),* $(,)?]),+ $(,)?) => {
|
||||
paste::paste! {
|
||||
$(
|
||||
#[tokio::test]
|
||||
#[file_serial([<hf_ $case>])]
|
||||
$(#[$case_attr])*
|
||||
async fn [<roundtrip_ $case>]() -> Result<()> {
|
||||
let case = RoundtripCase::$case();
|
||||
let backends = load_roundtrip_backends(&case).await?;
|
||||
$(
|
||||
$(#[$fixture_attr])*
|
||||
[<run_roundtrip_ $fixture>](&case, &backends).await?;
|
||||
)*
|
||||
Ok(())
|
||||
@@ -300,6 +313,7 @@ roundtrip_tests! {
|
||||
gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call
|
||||
kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history
|
||||
gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call
|
||||
inkling => [reasoning_and_content, tool_call_mix],
|
||||
}
|
||||
|
||||
/// Run the fixed reasoning+content fixture for one model/parser case.
|
||||
|
||||
@@ -646,7 +646,7 @@ fn serve_args_reject_unknown_renderer_value() {
|
||||
.unwrap_err();
|
||||
|
||||
expect![[r#"
|
||||
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)
|
||||
error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling)
|
||||
|
||||
For more information, try '--help'.
|
||||
"#]]
|
||||
|
||||
@@ -31,24 +31,24 @@ use super::utils::{
|
||||
};
|
||||
use super::{Result, ToolCallDelta, ToolParserOutput};
|
||||
|
||||
type JsonToolInput<'i> = Partial<&'i str>;
|
||||
pub(crate) type JsonToolInput<'i> = Partial<&'i str>;
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct JsonToolCallConfig {
|
||||
parser_name: &'static str,
|
||||
start_marker: &'static str,
|
||||
end_marker: &'static str,
|
||||
marker_whitespace: JsonToolCallWhitespace,
|
||||
delimiter: Option<&'static str>,
|
||||
name_key: &'static str,
|
||||
pub(crate) struct JsonToolCallConfig {
|
||||
pub parser_name: &'static str,
|
||||
pub start_marker: &'static str,
|
||||
pub end_marker: &'static str,
|
||||
pub marker_whitespace: JsonToolCallWhitespace,
|
||||
pub delimiter: Option<&'static str>,
|
||||
pub name_key: &'static str,
|
||||
/// Candidate JSON keys naming the arguments payload, tried in order.
|
||||
/// Most parsers use a single key like `["arguments"]`, but some accept
|
||||
/// multiple (e.g. InternLM2 accepts `parameters` or `arguments`).
|
||||
arguments_key: &'static [&'static str],
|
||||
pub arguments_key: &'static [&'static str],
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum JsonToolCallWhitespace {
|
||||
pub(crate) enum JsonToolCallWhitespace {
|
||||
Optional,
|
||||
Exact(&'static str),
|
||||
}
|
||||
@@ -61,7 +61,7 @@ enum JsonToolCallMode {
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
enum JsonToolCallEvent {
|
||||
pub(crate) enum JsonToolCallEvent {
|
||||
Text { len: usize },
|
||||
ToolCallStart,
|
||||
ToolCallHeader { function_name: String },
|
||||
@@ -220,7 +220,7 @@ fn tool_call_start_event(
|
||||
|
||||
/// Parse a marker-wrapped JSON tool-call header before the raw arguments
|
||||
/// payload.
|
||||
fn tool_call_header_event(
|
||||
pub(crate) fn tool_call_header_event(
|
||||
input: &mut JsonToolInput<'_>,
|
||||
config: JsonToolCallConfig,
|
||||
) -> ModalResult<JsonToolCallEvent> {
|
||||
|
||||
@@ -9,7 +9,7 @@ mod deepseek_dsml;
|
||||
pub(crate) mod deepseek_json;
|
||||
mod glm_xml;
|
||||
mod hy_v3;
|
||||
mod json;
|
||||
pub(crate) mod json;
|
||||
mod kimi_k2;
|
||||
mod minimax_m2;
|
||||
mod minimax_m3;
|
||||
|
||||
@@ -10,7 +10,7 @@ use winnow::prelude::*;
|
||||
use winnow::stream::{Partial, Stream};
|
||||
use winnow::token::{literal, take_till, take_until};
|
||||
|
||||
use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
|
||||
use super::{Result, UnifiedParser, UnifiedParserOutput, token_id};
|
||||
use crate::reasoning::last_reasoning_boundary;
|
||||
use crate::tool::{Tool, ToolCallDelta};
|
||||
use crate::unified::parsing_failed;
|
||||
@@ -79,17 +79,8 @@ pub struct Gemma4UnifiedParser {
|
||||
impl Gemma4UnifiedParser {
|
||||
/// Create a Gemma4 parser.
|
||||
pub fn new(_tools: &[Tool], tokenizer: DynTokenizer) -> Result<Self> {
|
||||
let channel_start_token_id = tokenizer.token_to_id(CHANNEL_START).ok_or_else(|| {
|
||||
UnifiedParserError::MissingToken {
|
||||
token: CHANNEL_START.to_string(),
|
||||
}
|
||||
})?;
|
||||
let channel_end_token_id =
|
||||
tokenizer
|
||||
.token_to_id(CHANNEL_END)
|
||||
.ok_or_else(|| UnifiedParserError::MissingToken {
|
||||
token: CHANNEL_END.to_string(),
|
||||
})?;
|
||||
let channel_start_token_id = token_id(tokenizer.as_ref(), CHANNEL_START)?;
|
||||
let channel_end_token_id = token_id(tokenizer.as_ref(), CHANNEL_END)?;
|
||||
|
||||
Ok(Self {
|
||||
buffer: String::new(),
|
||||
@@ -524,10 +515,10 @@ mod tests {
|
||||
|
||||
use super::{
|
||||
CHANNEL_END, CHANNEL_START, Gemma4UnifiedParser, ToolCallDelta, UnifiedParser,
|
||||
UnifiedParserError, UnifiedParserOutput, gemma4_array_content, parse_gemma4_args,
|
||||
UnifiedParserOutput, gemma4_array_content, parse_gemma4_args,
|
||||
};
|
||||
use crate::tool::Tool;
|
||||
use crate::unified::{UnifiedParserEvent, parsing_failed};
|
||||
use crate::unified::{UnifiedParserError, UnifiedParserEvent, parsing_failed};
|
||||
|
||||
const CHANNEL_START_ID: u32 = 256;
|
||||
const CHANNEL_END_ID: u32 = 257;
|
||||
|
||||
@@ -0,0 +1,768 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use winnow::ascii::multispace0 as ws0;
|
||||
use winnow::combinator::{alt, seq};
|
||||
use winnow::error::ModalResult;
|
||||
use winnow::prelude::*;
|
||||
use winnow::stream::Partial;
|
||||
use winnow::token::literal;
|
||||
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use super::{Result, UnifiedParser, UnifiedParserOutput, token_id};
|
||||
use crate::tool::json::{
|
||||
JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput,
|
||||
tool_call_header_event,
|
||||
};
|
||||
use crate::tool::{Tool, ToolCallDelta};
|
||||
use crate::unified::parsing_failed;
|
||||
use crate::utils::{
|
||||
JsonObjectScanState, parse_buffered_event, safe_text_len_mul, take_json_object,
|
||||
};
|
||||
|
||||
const CONTENT_TEXT: &str = "<|content_text|>";
|
||||
const CONTENT_THINKING: &str = "<|content_thinking|>";
|
||||
const CONTENT_INVOKE_TOOL_JSON: &str = "<|content_invoke_tool_json|>";
|
||||
const CONTENT_INVOKE_TOOL_TEXT: &str = "<|content_invoke_tool_text|>";
|
||||
const CONTENT_MODEL_END_SAMPLING: &str = "<|content_model_end_sampling|>";
|
||||
const CONTENT_TOOL_ERROR: &str = "<|content_tool_error|>";
|
||||
const MESSAGE_MODEL: &str = "<|message_model|>";
|
||||
const END_MESSAGE: &str = "<|end_message|>";
|
||||
|
||||
const IDLE_MARKERS: &[&str] = &[
|
||||
MESSAGE_MODEL,
|
||||
CONTENT_TEXT,
|
||||
CONTENT_THINKING,
|
||||
CONTENT_INVOKE_TOOL_JSON,
|
||||
CONTENT_INVOKE_TOOL_TEXT,
|
||||
CONTENT_MODEL_END_SAMPLING,
|
||||
CONTENT_TOOL_ERROR,
|
||||
END_MESSAGE,
|
||||
];
|
||||
const BLOCK_END_MARKERS: &[&str] = &[END_MESSAGE, CONTENT_MODEL_END_SAMPLING];
|
||||
|
||||
const INKLING_TOOL_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
|
||||
parser_name: "Inkling",
|
||||
start_marker: CONTENT_INVOKE_TOOL_JSON,
|
||||
end_marker: END_MESSAGE,
|
||||
marker_whitespace: JsonToolCallWhitespace::Optional,
|
||||
delimiter: None,
|
||||
name_key: "name",
|
||||
arguments_key: &["args"],
|
||||
};
|
||||
|
||||
type InklingInput<'i> = Partial<&'i str>;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
enum InklingEvent {
|
||||
Text { len: usize },
|
||||
Reasoning { len: usize },
|
||||
TextStart,
|
||||
ReasoningStart,
|
||||
MessageStart,
|
||||
Header,
|
||||
ToolJsonStart,
|
||||
ToolJsonHeader { name: String },
|
||||
ToolJsonArgs { len: usize, complete: bool },
|
||||
BlockEnd,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
enum InklingMode {
|
||||
#[default]
|
||||
Idle,
|
||||
MessageHeader,
|
||||
Text,
|
||||
Reasoning,
|
||||
ToolJsonHeader,
|
||||
ToolJsonArgs {
|
||||
json_scan: JsonObjectScanState,
|
||||
},
|
||||
ToolJsonClose,
|
||||
}
|
||||
|
||||
/// Unified parser for Inkling typed content blocks.
|
||||
pub struct InklingUnifiedParser {
|
||||
buffer: String,
|
||||
mode: InklingMode,
|
||||
emitted_tool_count: usize,
|
||||
active_tool_index: Option<usize>,
|
||||
tokenizer: DynTokenizer,
|
||||
message_model_token_id: u32,
|
||||
content_text_token_id: u32,
|
||||
content_thinking_token_id: u32,
|
||||
}
|
||||
|
||||
impl InklingUnifiedParser {
|
||||
/// Create a Inkling parser.
|
||||
pub fn new(_tools: &[Tool], tokenizer: DynTokenizer) -> Result<Self> {
|
||||
let message_model_token_id = token_id(tokenizer.as_ref(), MESSAGE_MODEL)?;
|
||||
let content_text_token_id = token_id(tokenizer.as_ref(), CONTENT_TEXT)?;
|
||||
let content_thinking_token_id = token_id(tokenizer.as_ref(), CONTENT_THINKING)?;
|
||||
|
||||
Ok(Self {
|
||||
buffer: String::new(),
|
||||
mode: InklingMode::Idle,
|
||||
emitted_tool_count: 0,
|
||||
active_tool_index: None,
|
||||
tokenizer,
|
||||
message_model_token_id,
|
||||
content_text_token_id,
|
||||
content_thinking_token_id,
|
||||
})
|
||||
}
|
||||
|
||||
fn initialize_mode(&mut self, prompt_token_ids: &[u32]) {
|
||||
self.mode = InklingMode::Idle;
|
||||
for token_id in prompt_token_ids.iter().rev().copied() {
|
||||
if token_id == self.message_model_token_id {
|
||||
self.mode = InklingMode::MessageHeader;
|
||||
return;
|
||||
}
|
||||
if token_id == self.content_thinking_token_id {
|
||||
self.mode = InklingMode::Reasoning;
|
||||
return;
|
||||
}
|
||||
if token_id == self.content_text_token_id {
|
||||
self.mode = InklingMode::Text;
|
||||
return;
|
||||
}
|
||||
if self.tokenizer.is_special_id(token_id) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_event(&mut self, event: InklingEvent, output: &mut UnifiedParserOutput) -> Result<()> {
|
||||
match event {
|
||||
InklingEvent::Text { len } => output.push_text(self.buffer[..len].to_string()),
|
||||
InklingEvent::Reasoning { len } => {
|
||||
output.push_reasoning(self.buffer[..len].to_string());
|
||||
}
|
||||
InklingEvent::MessageStart => self.mode = InklingMode::MessageHeader,
|
||||
InklingEvent::Header => {}
|
||||
InklingEvent::TextStart => self.mode = InklingMode::Text,
|
||||
InklingEvent::ReasoningStart => self.mode = InklingMode::Reasoning,
|
||||
InklingEvent::ToolJsonStart => self.mode = InklingMode::ToolJsonHeader,
|
||||
InklingEvent::ToolJsonHeader { name } => {
|
||||
let tool_index = self.emitted_tool_count;
|
||||
self.emitted_tool_count += 1;
|
||||
self.active_tool_index = Some(tool_index);
|
||||
self.mode = InklingMode::ToolJsonArgs {
|
||||
json_scan: JsonObjectScanState::default(),
|
||||
};
|
||||
output.push_call(ToolCallDelta {
|
||||
tool_index,
|
||||
name: Some(name),
|
||||
arguments: String::new(),
|
||||
});
|
||||
}
|
||||
InklingEvent::ToolJsonArgs { len, complete } => {
|
||||
let Some(tool_index) = self.active_tool_index else {
|
||||
return Err(parsing_failed!(
|
||||
"Inkling arguments without an active tool call"
|
||||
));
|
||||
};
|
||||
output.push_call(ToolCallDelta {
|
||||
tool_index,
|
||||
name: None,
|
||||
arguments: self.buffer[..len].to_string(),
|
||||
});
|
||||
if complete {
|
||||
self.mode = InklingMode::ToolJsonClose;
|
||||
}
|
||||
}
|
||||
InklingEvent::BlockEnd => {
|
||||
self.mode = InklingMode::Idle;
|
||||
self.active_tool_index = None;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> String {
|
||||
self.mode = InklingMode::Idle;
|
||||
self.active_tool_index = None;
|
||||
self.emitted_tool_count = 0;
|
||||
std::mem::take(&mut self.buffer)
|
||||
}
|
||||
}
|
||||
|
||||
impl UnifiedParser for InklingUnifiedParser {
|
||||
fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Self::new(tools, tokenizer).map(|parser| Box::new(parser) as Box<dyn UnifiedParser>)
|
||||
}
|
||||
|
||||
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
|
||||
self.buffer.clear();
|
||||
self.emitted_tool_count = 0;
|
||||
self.active_tool_index = None;
|
||||
self.initialize_mode(prompt_token_ids);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn preserve_special_tokens(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, chunk: &str, output: &mut UnifiedParserOutput) -> Result<()> {
|
||||
self.buffer.push_str(chunk);
|
||||
|
||||
while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| {
|
||||
parse_next_inkling_event(input, &mut self.mode)
|
||||
})? {
|
||||
self.apply_event(event, output)?;
|
||||
self.buffer.drain(..consumed_len);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Result<UnifiedParserOutput> {
|
||||
let mut output = UnifiedParserOutput::default();
|
||||
|
||||
match &self.mode {
|
||||
InklingMode::Idle | InklingMode::Text => {
|
||||
output.push_text(std::mem::take(&mut self.buffer))
|
||||
}
|
||||
InklingMode::MessageHeader => self.buffer.clear(),
|
||||
InklingMode::Reasoning => output.push_reasoning(std::mem::take(&mut self.buffer)),
|
||||
InklingMode::ToolJsonHeader
|
||||
| InklingMode::ToolJsonArgs { .. }
|
||||
| InklingMode::ToolJsonClose => {
|
||||
return Err(parsing_failed!("incomplete Inkling tool call"));
|
||||
}
|
||||
}
|
||||
|
||||
let _ = self.reset();
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> String {
|
||||
InklingUnifiedParser::reset(self)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse one Inkling event from buffered streaming input.
|
||||
fn parse_next_inkling_event(
|
||||
input: &mut InklingInput<'_>,
|
||||
mode: &mut InklingMode,
|
||||
) -> ModalResult<InklingEvent> {
|
||||
match mode {
|
||||
InklingMode::Idle => parse_idle_event(input),
|
||||
InklingMode::MessageHeader => parse_message_header_event(input),
|
||||
InklingMode::Text => parse_text_event(input),
|
||||
InklingMode::Reasoning => parse_reasoning_event(input),
|
||||
InklingMode::ToolJsonHeader => parse_tool_json_header_event(input),
|
||||
InklingMode::ToolJsonArgs { json_scan } => parse_tool_json_args_event(input, json_scan),
|
||||
InklingMode::ToolJsonClose => parse_tool_json_close_event(input),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse an event while waiting for a Inkling content kind.
|
||||
fn parse_idle_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
alt((
|
||||
message_start_event,
|
||||
reasoning_start_event,
|
||||
text_start_event,
|
||||
tool_json_start_event,
|
||||
raw_text_start_event,
|
||||
block_end_event,
|
||||
safe_idle_text_event,
|
||||
))
|
||||
.parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse a Inkling model-authored message start marker.
|
||||
fn message_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
literal(MESSAGE_MODEL).value(InklingEvent::MessageStart).parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse an event while waiting for an Inkling message content kind.
|
||||
fn parse_message_header_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
alt((
|
||||
reasoning_start_event,
|
||||
text_start_event,
|
||||
tool_json_start_event,
|
||||
raw_text_start_event,
|
||||
block_end_event,
|
||||
safe_header_event,
|
||||
))
|
||||
.parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse an event inside a Inkling text block.
|
||||
fn parse_text_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
alt((block_end_event, safe_text_event)).parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse an event inside a Inkling reasoning block.
|
||||
fn parse_reasoning_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
alt((block_end_event, safe_reasoning_event)).parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse a Inkling text start marker.
|
||||
fn text_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
literal(CONTENT_TEXT).value(InklingEvent::TextStart).parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse a Inkling reasoning start marker.
|
||||
fn reasoning_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
literal(CONTENT_THINKING).value(InklingEvent::ReasoningStart).parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse a Inkling JSON tool-call start marker.
|
||||
fn tool_json_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
literal(CONTENT_INVOKE_TOOL_JSON)
|
||||
.value(InklingEvent::ToolJsonStart)
|
||||
.parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse a Inkling content kind treated as visible text.
|
||||
fn raw_text_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
alt((
|
||||
literal(CONTENT_INVOKE_TOOL_TEXT),
|
||||
literal(CONTENT_TOOL_ERROR),
|
||||
))
|
||||
.value(InklingEvent::TextStart)
|
||||
.parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse a Inkling block end marker.
|
||||
fn block_end_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
alt((literal(END_MESSAGE), literal(CONTENT_MODEL_END_SAMPLING)))
|
||||
.value(InklingEvent::BlockEnd)
|
||||
.parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse safe text while waiting for the next Inkling marker.
|
||||
fn safe_idle_text_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
safe_text_len_mul(input, IDLE_MARKERS).map(|len| InklingEvent::Text { len })
|
||||
}
|
||||
|
||||
/// Parse safe header text before the next Inkling marker.
|
||||
fn safe_header_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
safe_text_len_mul(input, IDLE_MARKERS).map(|_| InklingEvent::Header)
|
||||
}
|
||||
|
||||
/// Parse safe text before the end of a Inkling text block.
|
||||
fn safe_text_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
safe_text_len_mul(input, BLOCK_END_MARKERS).map(|len| InklingEvent::Text { len })
|
||||
}
|
||||
|
||||
/// Parse safe reasoning before the end of a Inkling reasoning block.
|
||||
fn safe_reasoning_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
safe_text_len_mul(input, BLOCK_END_MARKERS).map(|len| InklingEvent::Reasoning { len })
|
||||
}
|
||||
|
||||
/// Parse a Inkling JSON tool-call header.
|
||||
fn parse_tool_json_header_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
match tool_call_header_event(input, INKLING_TOOL_CONFIG)? {
|
||||
JsonToolCallEvent::ToolCallHeader { function_name } => Ok(InklingEvent::ToolJsonHeader {
|
||||
name: function_name,
|
||||
}),
|
||||
_ => unreachable!("tool_call_header_event only emits ToolCallHeader"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse raw Inkling JSON tool-call argument bytes.
|
||||
fn parse_tool_json_args_event(
|
||||
input: &mut JsonToolInput<'_>,
|
||||
json_scan: &mut JsonObjectScanState,
|
||||
) -> ModalResult<InklingEvent> {
|
||||
let len = take_json_object(input, json_scan)?;
|
||||
Ok(InklingEvent::ToolJsonArgs {
|
||||
len,
|
||||
complete: json_scan.complete(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse the close of a Inkling JSON tool-call block.
|
||||
fn parse_tool_json_close_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
|
||||
seq!(
|
||||
_: ws0,
|
||||
_: literal("}"),
|
||||
_: ws0,
|
||||
_: literal(END_MESSAGE),
|
||||
)
|
||||
.value(InklingEvent::BlockEnd)
|
||||
.parse_next(input)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::{CONTENT_TEXT, CONTENT_THINKING, InklingUnifiedParser, MESSAGE_MODEL};
|
||||
use crate::tool::Tool;
|
||||
use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
|
||||
use thiserror_ext::AsReport;
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
MESSAGE_MODEL => Some(200001),
|
||||
"<|content_text|>" => Some(200004),
|
||||
"<|content_model_end_sampling|>" => Some(200006),
|
||||
"<|content_thinking|>" => Some(200008),
|
||||
"<|end_message|>" => Some(200010),
|
||||
"<|content_tool_error|>" => Some(200022),
|
||||
"<|content_invoke_tool_json|>" => Some(200049),
|
||||
"<|content_invoke_tool_text|>" => Some(200057),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
let token = match id {
|
||||
200001 => MESSAGE_MODEL,
|
||||
200004 => "<|content_text|>",
|
||||
200006 => "<|content_model_end_sampling|>",
|
||||
200008 => "<|content_thinking|>",
|
||||
200010 => "<|end_message|>",
|
||||
200022 => "<|content_tool_error|>",
|
||||
200049 => "<|content_invoke_tool_json|>",
|
||||
200057 => "<|content_invoke_tool_text|>",
|
||||
_ => return None,
|
||||
};
|
||||
Some(token.to_string())
|
||||
}
|
||||
|
||||
fn is_special_id(&self, token_id: u32) -> bool {
|
||||
(199999..=200057).contains(&token_id)
|
||||
}
|
||||
}
|
||||
|
||||
trait UnifiedParserTestExt {
|
||||
fn parse_chunk(&mut self, chunk: &str) -> super::Result<UnifiedParserOutput>;
|
||||
fn parse_complete(&mut self, text: &str) -> super::Result<UnifiedParserOutput>;
|
||||
}
|
||||
|
||||
impl<T: UnifiedParser + ?Sized> UnifiedParserTestExt for T {
|
||||
fn parse_chunk(&mut self, chunk: &str) -> super::Result<UnifiedParserOutput> {
|
||||
let mut output = UnifiedParserOutput::default();
|
||||
self.parse_into(chunk, &mut output)?;
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn parse_complete(&mut self, text: &str) -> super::Result<UnifiedParserOutput> {
|
||||
let mut output = self.parse_chunk(text)?;
|
||||
output.append(self.finish()?);
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
trait UnifiedParserOutputTestExt {
|
||||
fn normal_text(&self) -> String;
|
||||
fn reasoning_text(&self) -> String;
|
||||
fn calls(&self) -> Vec<crate::tool::ToolCallDelta>;
|
||||
}
|
||||
|
||||
impl UnifiedParserOutputTestExt for UnifiedParserOutput {
|
||||
fn normal_text(&self) -> String {
|
||||
self.events
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
UnifiedParserEvent::Text(text) => Some(text.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn reasoning_text(&self) -> String {
|
||||
self.events
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
UnifiedParserEvent::Reasoning(text) => Some(text.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn calls(&self) -> Vec<crate::tool::ToolCallDelta> {
|
||||
self.events
|
||||
.iter()
|
||||
.filter_map(|event| match event {
|
||||
UnifiedParserEvent::ToolCall(call) => Some(call.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn test_tools() -> Vec<Tool> {
|
||||
vec![Tool {
|
||||
name: "get_weather".to_string(),
|
||||
description: Some("Get weather".to_string()),
|
||||
parameters: serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"}
|
||||
},
|
||||
}),
|
||||
strict: None,
|
||||
}]
|
||||
}
|
||||
|
||||
fn test_parser() -> InklingUnifiedParser {
|
||||
InklingUnifiedParser::new(&test_tools(), Arc::new(FakeTokenizer)).unwrap()
|
||||
}
|
||||
|
||||
fn collect_stream(chunks: &[&str]) -> UnifiedParserOutput {
|
||||
let mut parser = test_parser();
|
||||
let mut output = UnifiedParserOutput::default();
|
||||
for chunk in chunks {
|
||||
output.append(parser.parse_chunk(chunk).unwrap());
|
||||
}
|
||||
output.append(parser.finish().unwrap());
|
||||
output
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_streaming_emits_reasoning_then_text() {
|
||||
let output = collect_stream(&[concat!(
|
||||
"<|content_thinking|>reason<|end_message|>",
|
||||
"<|message_model|><|content_text|>answer<|end_message|>",
|
||||
"<|content_model_end_sampling|>"
|
||||
)]);
|
||||
|
||||
assert_eq!(output.reasoning_text(), "reason");
|
||||
assert_eq!(output.normal_text(), "answer");
|
||||
assert!(output.calls().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_streaming_holds_split_markers() {
|
||||
let output = collect_stream(&[
|
||||
"<|content_thin",
|
||||
"king|>rea",
|
||||
"son<|end_mes",
|
||||
"sage|><|message_",
|
||||
"model|><|content_text|>answer",
|
||||
"<|end_message|>",
|
||||
]);
|
||||
|
||||
assert_eq!(output.reasoning_text(), "reason");
|
||||
assert_eq!(output.normal_text(), "answer");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_streaming_accepts_content_blocks_without_message_start() {
|
||||
let output = collect_stream(&[concat!(
|
||||
"<|content_thinking|>reason<|end_message|>",
|
||||
"<|content_text|>answer<|end_message|>",
|
||||
)]);
|
||||
|
||||
assert_eq!(output.reasoning_text(), "reason");
|
||||
assert_eq!(output.normal_text(), "answer");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_tool_json_streams_name_then_argument_deltas() {
|
||||
let mut parser = test_parser();
|
||||
let chunks = [
|
||||
"<|message_model|><|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":",
|
||||
"{\"city\":",
|
||||
"\"SF\"",
|
||||
"}}<|end_message|>",
|
||||
];
|
||||
|
||||
let mut output = UnifiedParserOutput::default();
|
||||
let mut observed_args = Vec::new();
|
||||
for chunk in chunks {
|
||||
let next = parser.parse_chunk(chunk).unwrap();
|
||||
observed_args.extend(
|
||||
next.calls()
|
||||
.into_iter()
|
||||
.filter(|call| call.name.is_none())
|
||||
.map(|call| call.arguments),
|
||||
);
|
||||
output.append(next);
|
||||
}
|
||||
output.append(parser.finish().unwrap());
|
||||
|
||||
let calls = output.calls();
|
||||
assert_eq!(calls[0].name.as_deref(), Some("get_weather"));
|
||||
assert_eq!(observed_args, ["{\"city\":", "\"SF\"", "}"]);
|
||||
assert_eq!(
|
||||
calls.iter().map(|call| call.arguments.as_str()).collect::<String>(),
|
||||
"{\"city\":\"SF\"}"
|
||||
);
|
||||
assert!(output.normal_text().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_discards_tool_name_from_message_header() {
|
||||
let output = collect_stream(&[concat!(
|
||||
"<|message_model|>get_weather<|content_invoke_tool_json|>",
|
||||
"{\"name\":\"get_weather\",\"args\":{}}<|end_message|>"
|
||||
)]);
|
||||
|
||||
assert!(output.normal_text().is_empty());
|
||||
assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_streaming_handles_multiple_tool_blocks() {
|
||||
let output = collect_stream(&[concat!(
|
||||
"<|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":{\"city\":\"SF\"}}<|end_message|>",
|
||||
"<|message_model|><|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":{\"city\":\"NYC\"}}<|end_message|>"
|
||||
)]);
|
||||
|
||||
let calls = output.calls();
|
||||
assert_eq!(calls.iter().filter(|call| call.name.is_some()).count(), 2);
|
||||
assert_eq!(calls[0].tool_index, 0);
|
||||
assert_eq!(calls[2].tool_index, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_initialize_open_text_prompt_starts_in_text() {
|
||||
let mut parser = test_parser();
|
||||
parser.initialize(&[200004]).unwrap();
|
||||
|
||||
let output = parser.parse_complete("answer<|end_message|>").unwrap();
|
||||
|
||||
assert_eq!(output.normal_text(), "answer");
|
||||
assert!(output.reasoning_text().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_initialize_open_reasoning_prompt_starts_in_reasoning() {
|
||||
let mut parser = test_parser();
|
||||
parser.initialize(&[200008]).unwrap();
|
||||
|
||||
let output = parser.parse_complete("reason<|end_message|>").unwrap();
|
||||
|
||||
assert_eq!(output.reasoning_text(), "reason");
|
||||
assert!(output.normal_text().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_initialize_model_opener_starts_in_message_header() {
|
||||
let mut parser = test_parser();
|
||||
parser.initialize(&[200001]).unwrap();
|
||||
|
||||
let output = parser
|
||||
.parse_complete(concat!(
|
||||
"get_weather<|content_invoke_tool_json|>",
|
||||
"{\"name\":\"get_weather\",\"args\":{}}<|end_message|>"
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
assert!(output.normal_text().is_empty());
|
||||
assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_plain_text_falls_through_as_text() {
|
||||
let output = collect_stream(&["plain ", "answer"]);
|
||||
|
||||
assert_eq!(output.normal_text(), "plain answer");
|
||||
assert!(output.reasoning_text().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_raw_tool_text_and_tool_error_are_visible_text() {
|
||||
let output = collect_stream(&[concat!(
|
||||
"<|content_invoke_tool_text|>search SF<|end_message|>",
|
||||
"<|content_tool_error|>failed<|end_message|>"
|
||||
)]);
|
||||
|
||||
assert_eq!(output.normal_text(), "search SFfailed");
|
||||
assert!(output.calls().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_finish_fails_incomplete_tool_call() {
|
||||
let mut parser = test_parser();
|
||||
parser
|
||||
.parse_chunk("<|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":{\"city\"")
|
||||
.unwrap();
|
||||
|
||||
let error = parser.finish().unwrap_err();
|
||||
|
||||
assert!(error.as_report().to_string().contains("incomplete Inkling tool call"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_rejects_non_object_args() {
|
||||
let mut parser = test_parser();
|
||||
let error = parser
|
||||
.parse_chunk("<|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":42")
|
||||
.unwrap_err();
|
||||
|
||||
assert!(error.as_report().to_string().contains("JSON object argument"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inkling_missing_token_fails_create() {
|
||||
struct MissingTokenizer;
|
||||
|
||||
impl Tokenizer for MissingTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
_text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(String::new())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
MESSAGE_MODEL => Some(200001),
|
||||
CONTENT_TEXT => Some(200004),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
match id {
|
||||
200001 => Some(MESSAGE_MODEL.to_string()),
|
||||
200004 => Some(CONTENT_TEXT.to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let error = match InklingUnifiedParser::new(&[], Arc::new(MissingTokenizer)) {
|
||||
Ok(_) => panic!("expected parser creation to fail"),
|
||||
Err(error) => error,
|
||||
};
|
||||
|
||||
assert!(error.as_report().to_string().contains(CONTENT_THINKING));
|
||||
}
|
||||
}
|
||||
@@ -5,9 +5,11 @@
|
||||
|
||||
mod combined;
|
||||
mod gemma4;
|
||||
mod inkling;
|
||||
|
||||
pub use combined::CombinedParser;
|
||||
pub use gemma4::Gemma4UnifiedParser;
|
||||
pub use inkling::InklingUnifiedParser;
|
||||
use thiserror::Error;
|
||||
use thiserror_ext::Macro;
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
@@ -208,3 +210,10 @@ pub enum UnifiedParserError {
|
||||
#[error(transparent)]
|
||||
Tool(#[from] ToolParserError),
|
||||
}
|
||||
|
||||
/// Returns the ID for the given token, or an error if it's not found.
|
||||
fn token_id(tokenizer: &dyn vllm_tokenizer::Tokenizer, token: &str) -> Result<u32> {
|
||||
tokenizer.token_to_id(token).ok_or_else(|| UnifiedParserError::MissingToken {
|
||||
token: token.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -95,6 +95,7 @@ impl HfSpecialTokens {
|
||||
pub struct ModelConfig {
|
||||
model_type: Option<String>,
|
||||
vocab_size: Option<u32>,
|
||||
eos_token_id: Option<OneOrManyTokenIds>,
|
||||
num_experts: Option<OneOrManyExpertCount>,
|
||||
moe_num_experts: Option<OneOrManyExpertCount>,
|
||||
n_routed_experts: Option<OneOrManyExpertCount>,
|
||||
@@ -125,12 +126,16 @@ pub(super) enum OneOrManyTokenIds {
|
||||
}
|
||||
|
||||
impl OneOrManyTokenIds {
|
||||
pub(super) fn into_set(self) -> BTreeSet<u32> {
|
||||
pub(super) fn as_slice(&self) -> &[u32] {
|
||||
match self {
|
||||
Self::One(id) => BTreeSet::from([id]),
|
||||
Self::Many(ids) => ids.into_iter().collect(),
|
||||
Self::One(id) => std::slice::from_ref(id),
|
||||
Self::Many(ids) => ids.as_slice(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn into_set(self) -> BTreeSet<u32> {
|
||||
self.as_slice().iter().copied().collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Hugging Face configs may expose the expert count either as one integer or
|
||||
@@ -195,6 +200,18 @@ impl ModelConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the effective model-side EOS token ids, following the same
|
||||
/// simplified text-config selection as `vocab_size`.
|
||||
pub(super) fn eos_token_ids(&self) -> &[u32] {
|
||||
if let Some(eos_token_id) = self.eos_token_id.as_ref() {
|
||||
eos_token_id.as_slice()
|
||||
} else if let Some(text_config) = self.text_config.as_deref() {
|
||||
text_config.eos_token_ids()
|
||||
} else {
|
||||
&[]
|
||||
}
|
||||
}
|
||||
|
||||
/// Match Python's current expert-count priority on the selected text
|
||||
/// config.
|
||||
///
|
||||
@@ -354,6 +371,32 @@ mod tests {
|
||||
assert_eq!(config.vocab_size().unwrap(), 151936);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_reads_top_level_eos_token_ids() {
|
||||
let single: ModelConfig = serde_json::from_str(r#"{"eos_token_id":151645}"#).unwrap();
|
||||
let many: ModelConfig =
|
||||
serde_json::from_str(r#"{"eos_token_id":[128001,128008,128009]}"#).unwrap();
|
||||
let null: ModelConfig = serde_json::from_str(r#"{"eos_token_id":null}"#).unwrap();
|
||||
|
||||
assert_eq!(single.eos_token_ids(), &[151645]);
|
||||
assert_eq!(many.eos_token_ids(), &[128001, 128008, 128009]);
|
||||
assert!(null.eos_token_ids().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_uses_nested_eos_token_ids_when_top_level_is_absent() {
|
||||
let config: ModelConfig = serde_json::from_str(
|
||||
r#"{
|
||||
"text_config": {
|
||||
"eos_token_id": [59246, 59253, 59255]
|
||||
}
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(config.eos_token_ids(), &[59246, 59253, 59255]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_rejects_missing_vocab_size() {
|
||||
let config: ModelConfig = serde_json::from_str(r#"{}"#).unwrap();
|
||||
|
||||
@@ -8,7 +8,9 @@ use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use tracing::info;
|
||||
use vllm_tokenizer::{DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer};
|
||||
use vllm_tokenizer::{
|
||||
DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer, Tokenizer,
|
||||
};
|
||||
|
||||
use self::config::{GenerationConfig, load_generation_config};
|
||||
pub use self::config::{
|
||||
@@ -56,23 +58,15 @@ impl HfTextBackend {
|
||||
pub fn from_resolved_model_files(files: ResolvedModelFiles, model_id: String) -> Result<Self> {
|
||||
let tokenizer_config = load_tokenizer_config(files.tokenizer_config_path.as_deref())?;
|
||||
let tokenizer = load_tokenizer(&files.tokenizer)?;
|
||||
let primary_eos_token_id = tokenizer_config
|
||||
.special_tokens
|
||||
.eos_token
|
||||
.as_ref()
|
||||
.and_then(|token| tokenizer.token_to_id(token.as_str()));
|
||||
|
||||
let model_config = load_model_config(files.config_path.as_deref())?;
|
||||
let model_vocab_size = model_config.vocab_size()? as usize;
|
||||
let generation_config = load_generation_config(files.generation_config_path.as_deref())?;
|
||||
let mut extra_eos_token_ids = generation_config
|
||||
.eos_token_id
|
||||
.clone()
|
||||
.map(|value| value.into_set())
|
||||
.unwrap_or_default();
|
||||
if let Some(primary_eos_token_id) = primary_eos_token_id {
|
||||
extra_eos_token_ids.remove(&primary_eos_token_id);
|
||||
}
|
||||
let (primary_eos_token_id, extra_eos_token_ids) = resolve_eos_token_ids(
|
||||
&tokenizer_config,
|
||||
&model_config,
|
||||
&generation_config,
|
||||
tokenizer.as_ref(),
|
||||
);
|
||||
|
||||
info!(
|
||||
model_id,
|
||||
@@ -98,6 +92,42 @@ impl HfTextBackend {
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve EOS hints from tokenizer, model, and generation configs.
|
||||
///
|
||||
/// Resolution rules:
|
||||
/// 1. Use the tokenizer-side EOS token as the primary EOS when it resolves to a
|
||||
/// token id.
|
||||
/// 2. Fall back to the first model-config EOS id when the tokenizer config does
|
||||
/// not provide a primary EOS.
|
||||
/// 3. Keep any remaining model/generation EOS ids as extra stop-token ids so
|
||||
/// they still participate in stopping and min-token handling.
|
||||
fn resolve_eos_token_ids(
|
||||
tokenizer_config: &HfTokenizerConfig,
|
||||
model_config: &ModelConfig,
|
||||
generation_config: &GenerationConfig,
|
||||
tokenizer: &dyn Tokenizer,
|
||||
) -> (Option<u32>, BTreeSet<u32>) {
|
||||
let model_config_eos_token_ids = model_config.eos_token_ids();
|
||||
let primary_eos_token_id = tokenizer_config
|
||||
.special_tokens
|
||||
.eos_token
|
||||
.as_ref()
|
||||
.and_then(|token| tokenizer.token_to_id(token.as_str()))
|
||||
.or_else(|| model_config_eos_token_ids.first().copied());
|
||||
|
||||
let mut extra_eos_token_ids = generation_config
|
||||
.eos_token_id
|
||||
.clone()
|
||||
.map(|value| value.into_set())
|
||||
.unwrap_or_default();
|
||||
extra_eos_token_ids.extend(model_config_eos_token_ids.iter().copied());
|
||||
if let Some(primary_eos_token_id) = primary_eos_token_id {
|
||||
extra_eos_token_ids.remove(&primary_eos_token_id);
|
||||
}
|
||||
|
||||
(primary_eos_token_id, extra_eos_token_ids)
|
||||
}
|
||||
|
||||
impl TextBackend for HfTextBackend {
|
||||
fn tokenizer(&self) -> DynTokenizer {
|
||||
self.tokenizer.clone()
|
||||
@@ -128,3 +158,77 @@ impl TextBackend for HfTextBackend {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::{GenerationConfig, HfTokenizerConfig, ModelConfig, resolve_eos_token_ids};
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
_text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(String::new())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
(token == "</s>").then_some(2)
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
(id == 2).then(|| "</s>".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eos_resolution_uses_model_config_when_tokenizer_has_no_eos() {
|
||||
let tokenizer_config: HfTokenizerConfig = serde_json::from_str("{}").unwrap();
|
||||
let model_config: ModelConfig =
|
||||
serde_json::from_str(r#"{"eos_token_id":[200006,200010]}"#).unwrap();
|
||||
let generation_config: GenerationConfig = serde_json::from_str("{}").unwrap();
|
||||
|
||||
let (primary, extra) = resolve_eos_token_ids(
|
||||
&tokenizer_config,
|
||||
&model_config,
|
||||
&generation_config,
|
||||
&FakeTokenizer,
|
||||
);
|
||||
|
||||
assert_eq!(primary, Some(200006));
|
||||
assert_eq!(extra, BTreeSet::from([200010]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eos_resolution_keeps_tokenizer_eos_primary() {
|
||||
let tokenizer_config: HfTokenizerConfig =
|
||||
serde_json::from_str(r#"{"eos_token":"</s>"}"#).unwrap();
|
||||
let model_config: ModelConfig =
|
||||
serde_json::from_str(r#"{"eos_token_id":[2,200006]}"#).unwrap();
|
||||
let generation_config: GenerationConfig =
|
||||
serde_json::from_str(r#"{"eos_token_id":[2,200010]}"#).unwrap();
|
||||
|
||||
let (primary, extra) = resolve_eos_token_ids(
|
||||
&tokenizer_config,
|
||||
&model_config,
|
||||
&generation_config,
|
||||
&FakeTokenizer,
|
||||
);
|
||||
|
||||
assert_eq!(primary, Some(2));
|
||||
assert_eq!(extra, BTreeSet::from([200006, 200010]));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user