diff --git a/.gitignore b/.gitignore index 26cd21a015..43787b8cd2 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ vllm/third_party/deep_gemm/ # fmha_sm100 vendored package built from source vllm/third_party/fmha_sm100/ +# tml-fa4 vendored package built from source +vllm/third_party/tml_fa4/ + # triton jit .triton diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ee11e6ba2..f514ba41aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1408,6 +1408,7 @@ if (VLLM_GPU_LANG STREQUAL "CUDA") include(cmake/external_projects/fmha_sm100.cmake) include(cmake/external_projects/flashmla.cmake) include(cmake/external_projects/qutlass.cmake) + include(cmake/external_projects/tml_fa4.cmake) # vllm-flash-attn should be last as it overwrites some CMake functions include(cmake/external_projects/vllm_flash_attn.cmake) diff --git a/cmake/external_projects/tml_fa4.cmake b/cmake/external_projects/tml_fa4.cmake new file mode 100644 index 0000000000..59e2e241c9 --- /dev/null +++ b/cmake/external_projects/tml_fa4.cmake @@ -0,0 +1,50 @@ +include(FetchContent) + +if(DEFINED ENV{TML_FA4_SRC_DIR}) + set(TML_FA4_SRC_DIR $ENV{TML_FA4_SRC_DIR}) +endif() + +if(TML_FA4_SRC_DIR) + FetchContent_Declare( + tml_fa4 + SOURCE_DIR ${TML_FA4_SRC_DIR} + CONFIGURE_COMMAND "" + BUILD_COMMAND "") +else() + FetchContent_Declare( + tml_fa4 + GIT_REPOSITORY https://github.com/vllm-project/tml-fa4.git + GIT_TAG 13374f0c855acc1add1bf30444bd67aebbc24a8e + GIT_PROGRESS TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "") +endif() + +FetchContent_GetProperties(tml_fa4) +if(NOT tml_fa4_POPULATED) + FetchContent_Populate(tml_fa4) +endif() +message(STATUS "tml-fa4 is available at ${tml_fa4_SOURCE_DIR}") + +add_custom_target(tml_fa4) + +# Install into a private namespace so this implementation cannot shadow the +# flash_attn package used by vLLM's standard attention backends. +install(CODE " + file(GLOB_RECURSE TML_FA4_PY_FILES + \"${tml_fa4_SOURCE_DIR}/flash_attn/cute/*.py\") + foreach(SRC_FILE \${TML_FA4_PY_FILES}) + file(RELATIVE_PATH REL_PATH + \"${tml_fa4_SOURCE_DIR}/flash_attn/cute\" \${SRC_FILE}) + set(DST_FILE + \"\${CMAKE_INSTALL_PREFIX}/vllm/third_party/tml_fa4/\${REL_PATH}\") + get_filename_component(DST_DIR \${DST_FILE} DIRECTORY) + file(MAKE_DIRECTORY \${DST_DIR}) + file(READ \${SRC_FILE} FILE_CONTENTS) + string(REPLACE + \"flash_attn.cute\" + \"vllm.third_party.tml_fa4\" + FILE_CONTENTS \"\${FILE_CONTENTS}\") + file(WRITE \${DST_FILE} \"\${FILE_CONTENTS}\") + endforeach() +" COMPONENT tml_fa4) diff --git a/requirements/cuda.txt b/requirements/cuda.txt index ab1df3ffe6..766acb0204 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -26,7 +26,7 @@ fastsafetensors >= 0.3.2 # QuACK and Cutlass DSL for FA4 (cute-DSL implementation) nvidia-cutlass-dsl[cu13]==4.5.2 -quack-kernels>=0.3.3 +quack-kernels>=0.4.0 # Required for tml-fa4 # Tokenspeed_MLA for faster mla with spec decode tokenspeed-mla==0.1.8; platform_system == "Linux" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 2a7b365ab4..378020f23f 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2220,7 +2220,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "llm-multimodal" version = "1.7.1" -source = "git+https://github.com/smg-project/llm-multimodal?rev=7df38e53f99aefaebe86934f010aa8084ec99b2f#7df38e53f99aefaebe86934f010aa8084ec99b2f" +source = "git+https://github.com/smg-project/llm-multimodal?rev=5390032d6dc8a3e6fdc83acd320260367eb4b9b5#5390032d6dc8a3e6fdc83acd320260367eb4b9b5" dependencies = [ "anyhow", "base64 0.22.1", @@ -2235,6 +2235,7 @@ dependencies = [ "once_cell", "pkg-config", "rayon", + "realfft", "reqwest 0.13.4", "rustfft", "serde", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 7c1793b448..a6a756fd88 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -53,7 +53,7 @@ hyper-util = { version = "0.1.20", features = [ indexmap = "2.13.0" itertools = "0.14.0" libc = "0.2.177" -llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "7df38e53f99aefaebe86934f010aa8084ec99b2f", default-features = false, features = ["native-tls"] } +llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "5390032d6dc8a3e6fdc83acd320260367eb4b9b5", default-features = false, features = ["native-tls"] } mimalloc = "0.1.52" minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] } minijinja-contrib = { version = "2.0", features = ["pycompat"] } diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index f938e1905c..d4999d07cd 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -20,6 +20,7 @@ use crate::output::{ use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo}; use crate::renderer::{ DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer, + InklingChatRenderer, }; use crate::request::ChatRequest; use crate::{DynChatOutputProcessor, RendererSelection}; @@ -71,6 +72,7 @@ impl HfChatBackend { RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()), RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()), RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?), + RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?), }; info!( diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 625a2d7818..7937afa130 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -33,7 +33,7 @@ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory}; pub use renderer::hf::ChatTemplateContentFormatOption; pub use renderer::{ ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, - HarmonyChatRenderer, RenderedPrompt, RendererSelection, + HarmonyChatRenderer, InklingChatRenderer, RenderedPrompt, RendererSelection, }; pub use request::{ ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool, @@ -294,7 +294,7 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string()); + expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string()); } #[test] @@ -305,6 +305,6 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); + expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); } } diff --git a/rust/src/chat/src/multimodal/audio.rs b/rust/src/chat/src/multimodal/audio.rs index 2759637754..4910f235ee 100644 --- a/rust/src/chat/src/multimodal/audio.rs +++ b/rust/src/chat/src/multimodal/audio.rs @@ -79,6 +79,38 @@ mod tests { use super::*; const AUDIO_PAD_ID: u32 = 151_676; + const INKLING_AUDIO_MARKER_ID: u32 = 200_020; + const INKLING_AUDIO_EMBED_ID: i32 = 200_053; + + fn inkling_info(decoder_dmodel: serde_json::Value) -> MultimodalModelInfo { + let config = serde_json::json!({ + "model_type": "inkling_mm_model", + "audio_config": { + "decoder_dmodel": decoder_dmodel, + "n_mel_bins": 80, + "mel_vocab_size": 16, + "dmel_min_value": -7.0, + "dmel_max_value": 2.0 + } + }); + let tokenizer = TestTokenizer::new() + .with_regular_token("<|content_image|>", 200_005) + .with_regular_token("<|content_audio_input|>", INKLING_AUDIO_MARKER_ID); + let context = MultimodalModelContext { + model_id: "inkling-test".to_string(), + model_type: Some("inkling_mm_model".to_string()), + config, + tokenizer: TokenizerResolver(Arc::new(tokenizer)), + }; + + MultimodalModelInfo::from_loaded( + context, + PreProcessorConfig::default(), + PreProcessorConfig::default(), + ) + .unwrap() + .expect("Inkling multimodal support") + } fn qwen3_asr_info() -> MultimodalModelInfo { let context = MultimodalModelContext { @@ -120,6 +152,40 @@ mod tests { bytes } + #[test] + fn resolves_inkling_audio_from_model_spec() { + let info = inkling_info(serde_json::json!(1024)); + let support = info.audio.as_ref().expect("audio support"); + + assert_eq!( + info.placeholder_token(Modality::Audio), + Some("<|content_audio_input|>") + ); + assert_eq!(support.placeholder.marker_token_id, INKLING_AUDIO_MARKER_ID); + assert_eq!( + support.placeholder.embed_token_id, + INKLING_AUDIO_EMBED_ID as u32 + ); + assert_eq!(support.spec.primary_key(), AUDIO_PRIMARY_KEY); + assert!(matches!( + &support.spec.field_layouts.encoder_input, + llm_multimodal::FieldLayout::Flat { sizes_key } + if sizes_key == "num_audio_tokens" + )); + assert!(matches!( + support.spec.field_layouts.model_specific.get("num_audio_tokens"), + Some(llm_multimodal::FieldLayout::Batched) + )); + } + + #[test] + fn inkling_decoder_config_gates_audio_capability() { + let info = inkling_info(serde_json::Value::Null); + + assert!(info.audio.is_none()); + assert_eq!(info.placeholder_token(Modality::Audio), None); + } + #[test] fn resolves_qwen_audio_from_model_spec() { let info = qwen3_asr_info(); @@ -210,4 +276,50 @@ mod tests { if tensor.dtype == "int64" && tensor.shape.is_empty() )); } + + #[tokio::test] + async fn inkling_tracker_and_processor_use_standard_audio_key() { + let info = inkling_info(serde_json::json!(1024)); + let wav = wav_i16_mono(16_000, &[0; 1_600]); + let expected_hash = llm_multimodal::hasher::hash_audio(&wav); + let fetched = info + .fetch_media(vec![MediaContentPart::AudioData { + data: wav, + mime_type: Some("audio/wav".to_string()), + uuid: Some("audio-1".to_string()), + }]) + .await + .unwrap(); + + let prepared = info.prepare_audios(fetched.audios, fetched.audio_uuids).await.unwrap(); + + assert_eq!(prepared.replacements.len(), 1); + assert_eq!( + prepared.replacements[0].tokens[0], + INKLING_AUDIO_MARKER_ID as i32 + ); + assert!( + prepared.replacements[0].tokens[1..] + .iter() + .all(|token| *token == INKLING_AUDIO_EMBED_ID) + ); + let item = &prepared.items[0]; + assert_eq!(item.hash, expected_hash); + assert_eq!(item.uuid.as_deref(), Some("audio-1")); + + let features = &item.data[AUDIO_PRIMARY_KEY]; + assert!(matches!(&features.field, MmField::Flat(_))); + assert!(matches!( + features.data.as_ref(), + Some(MmKwargValue::Tensor(tensor)) + if tensor.dtype == "float32" && tensor.shape.get(1) == Some(&80) + )); + let count = &item.data["num_audio_tokens"]; + assert!(matches!(&count.field, MmField::Batched(_))); + assert!(matches!( + count.data.as_ref(), + Some(MmKwargValue::Tensor(tensor)) + if tensor.dtype == "int64" && tensor.shape.is_empty() + )); + } } diff --git a/rust/src/chat/src/parser/reasoning/mod.rs b/rust/src/chat/src/parser/reasoning/mod.rs index 10d48b2b39..ca025268b0 100644 --- a/rust/src/chat/src/parser/reasoning/mod.rs +++ b/rust/src/chat/src/parser/reasoning/mod.rs @@ -23,6 +23,7 @@ pub mod names { pub const DEEPSEEK_V3: &str = "deepseek_v3"; pub const DEEPSEEK_V4: &str = "deepseek_v4"; pub const GEMMA4: &str = "gemma4"; + pub const INKLING: &str = "inkling"; pub const GLM45: &str = "glm45"; pub const KIMI: &str = "kimi"; pub const KIMI_K2: &str = "kimi_k2"; @@ -63,6 +64,7 @@ impl ReasoningParserFactory { .register_parser::(names::DEEPSEEK_V3) .register_parser::(names::DEEPSEEK_V4) .register_unified_dummy(names::GEMMA4) + .register_unified_dummy(names::INKLING) .register_parser::(names::GLM45) .register_parser::(names::KIMI) .register_parser::(names::KIMI_K2) diff --git a/rust/src/chat/src/parser/tool/mod.rs b/rust/src/chat/src/parser/tool/mod.rs index 3e394c417b..d8ca1fac8d 100644 --- a/rust/src/chat/src/parser/tool/mod.rs +++ b/rust/src/chat/src/parser/tool/mod.rs @@ -25,6 +25,7 @@ pub mod names { pub const GLM45: &str = "glm45"; pub const GLM47: &str = "glm47"; pub const GEMMA4: &str = "gemma4"; + pub const INKLING: &str = "inkling"; pub const GRANITE4: &str = "granite4"; pub const HERMES: &str = "hermes"; pub const HY_V3: &str = "hy_v3"; @@ -70,6 +71,7 @@ impl ToolParserFactory { .register_parser::(names::GLM45) .register_parser::(names::GLM47) .register_unified_dummy(names::GEMMA4) + .register_unified_dummy(names::INKLING) .register_parser::(names::GRANITE4) .register_parser::(names::HERMES) .register_parser::(names::HY_V3) diff --git a/rust/src/chat/src/parser/unified.rs b/rust/src/chat/src/parser/unified.rs index ff97bb38db..2536bf78c2 100644 --- a/rust/src/chat/src/parser/unified.rs +++ b/rust/src/chat/src/parser/unified.rs @@ -5,7 +5,7 @@ use std::sync::LazyLock; -pub use vllm_parser::unified::{Gemma4UnifiedParser, UnifiedParser}; +pub use vllm_parser::unified::{Gemma4UnifiedParser, InklingUnifiedParser, UnifiedParser}; use vllm_tokenizer::DynTokenizer; use crate::parser::ParserFactory; @@ -14,6 +14,7 @@ use crate::request::ChatTool; /// Canonical public names for registered unified parsers. pub mod names { pub const GEMMA4: &str = "gemma4"; + pub const INKLING: &str = "inkling"; } /// Constructor signature for one registered unified parser implementation. @@ -37,10 +38,12 @@ impl UnifiedParserFactory { let mut factory = Self::default(); factory.register_parser::(names::GEMMA4); + factory.register_parser::(names::INKLING); factory .register_pattern("gemma-4", names::GEMMA4) - .register_pattern("gemma4", names::GEMMA4); + .register_pattern("gemma4", names::GEMMA4) + .register_pattern("inkling", names::INKLING); factory } @@ -88,6 +91,13 @@ mod tests { .with_regular_token("", 257) } + fn inkling_tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_regular_token("<|message_model|>", 200001) + .with_regular_token("<|content_text|>", 200004) + .with_regular_token("<|content_thinking|>", 200008) + } + #[test] fn factory_registers_gemma4() { let factory = UnifiedParserFactory::new(); @@ -99,4 +109,16 @@ mod tests { ); factory.create(names::GEMMA4, &[], Arc::new(tokenizer())).unwrap(); } + + #[test] + fn factory_registers_inkling() { + let factory = UnifiedParserFactory::new(); + + assert!(factory.contains(names::INKLING)); + assert_eq!( + factory.resolve_name_for_model("thinkingmachines/Inkling"), + Some(names::INKLING) + ); + factory.create(names::INKLING, &[], Arc::new(inkling_tokenizer())).unwrap(); + } } diff --git a/rust/src/chat/src/renderer/inkling/fixtures/text_audio_input.json b/rust/src/chat/src/renderer/inkling/fixtures/text_audio_input.json new file mode 100644 index 0000000000..be704bdf1f --- /dev/null +++ b/rust/src/chat/src/renderer/inkling/fixtures/text_audio_input.json @@ -0,0 +1,21 @@ +{ + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "transcribe" + }, + { + "type": "input_audio", + "data": "" + }, + { + "type": "audio_url", + "audio_url": "data:audio/wav;base64," + } + ] + } + ] +} diff --git a/rust/src/chat/src/renderer/inkling/fixtures/text_audio_output.txt b/rust/src/chat/src/renderer/inkling/fixtures/text_audio_output.txt new file mode 100644 index 0000000000..659e2614e2 --- /dev/null +++ b/rust/src/chat/src/renderer/inkling/fixtures/text_audio_output.txt @@ -0,0 +1 @@ +<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>transcribe<|end_message|><|message_user|><|content_audio_input|><|audio_end|><|end_message|><|message_user|><|content_audio_input|><|audio_end|><|end_message|><|message_model|> diff --git a/rust/src/chat/src/renderer/inkling/fixtures/text_image_input.json b/rust/src/chat/src/renderer/inkling/fixtures/text_image_input.json new file mode 100644 index 0000000000..d0afac6401 --- /dev/null +++ b/rust/src/chat/src/renderer/inkling/fixtures/text_image_input.json @@ -0,0 +1,17 @@ +{ + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "look" + }, + { + "type": "image_url", + "image_url": "data:image/png;base64," + } + ] + } + ] +} diff --git a/rust/src/chat/src/renderer/inkling/fixtures/text_image_output.txt b/rust/src/chat/src/renderer/inkling/fixtures/text_image_output.txt new file mode 100644 index 0000000000..28a83a0ee8 --- /dev/null +++ b/rust/src/chat/src/renderer/inkling/fixtures/text_image_output.txt @@ -0,0 +1 @@ +<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>look<|end_message|><|message_user|><|content_image|><|end_message|><|message_model|> diff --git a/rust/src/chat/src/renderer/inkling/fixtures/tool_declare_input.json b/rust/src/chat/src/renderer/inkling/fixtures/tool_declare_input.json new file mode 100644 index 0000000000..a1ca1ca3cc --- /dev/null +++ b/rust/src/chat/src/renderer/inkling/fixtures/tool_declare_input.json @@ -0,0 +1,46 @@ +{ + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "required": [ + "city" + ], + "properties": { + "city": { + "type": "string" + } + } + } + } + } + ], + "messages": [ + { + "role": "developer", + "content": "rules", + "tools": [ + { + "type": "function", + "function": { + "name": "local_tool", + "parameters": { + "z": 1, + "a": { + "b": 2 + } + } + } + } + ] + }, + { + "role": "user", + "content": "hi" + } + ] +} diff --git a/rust/src/chat/src/renderer/inkling/fixtures/tool_declare_output.txt b/rust/src/chat/src/renderer/inkling/fixtures/tool_declare_output.txt new file mode 100644 index 0000000000..5a3c890325 --- /dev/null +++ b/rust/src/chat/src/renderer/inkling/fixtures/tool_declare_output.txt @@ -0,0 +1 @@ +<|message_system|>tool_declare<|content_xml|>[{"description":"Get weather information","name":"get_weather","parameters":{"properties":{"city":{"type":"string"}},"required":["city"],"type":"object"},"type":"function"},{"description":"","name":"local_tool","parameters":{"a":{"b":2},"z":1},"type":"function"}]<|end_message|><|message_system|><|content_text|>rules<|end_message|><|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>hi<|end_message|><|message_model|> diff --git a/rust/src/chat/src/renderer/inkling/fixtures/tool_round_trip_input.json b/rust/src/chat/src/renderer/inkling/fixtures/tool_round_trip_input.json new file mode 100644 index 0000000000..239e1348f1 --- /dev/null +++ b/rust/src/chat/src/renderer/inkling/fixtures/tool_round_trip_input.json @@ -0,0 +1,24 @@ +{ + "add_generation_prompt": false, + "messages": [ + { + "role": "assistant", + "reasoning_content": "think", + "content": "answer", + "tool_calls": [ + { + "id": "call_1", + "function": { + "name": "get_weather", + "arguments": "{\"city\":\"SF\"}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": "sunny" + } + ] +} diff --git a/rust/src/chat/src/renderer/inkling/fixtures/tool_round_trip_output.txt b/rust/src/chat/src/renderer/inkling/fixtures/tool_round_trip_output.txt new file mode 100644 index 0000000000..f5069f13be --- /dev/null +++ b/rust/src/chat/src/renderer/inkling/fixtures/tool_round_trip_output.txt @@ -0,0 +1 @@ +<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_model|><|content_thinking|>think<|end_message|><|message_model|><|content_text|>answer<|end_message|><|message_model|>get_weather<|content_invoke_tool_json|>{"name":"get_weather","args":{"city":"SF"}}<|end_message|><|content_model_end_sampling|><|message_tool|>get_weather<|content_text|>sunny<|end_message|> diff --git a/rust/src/chat/src/renderer/inkling/mod.rs b/rust/src/chat/src/renderer/inkling/mod.rs new file mode 100644 index 0000000000..8bf1a204fb --- /dev/null +++ b/rust/src/chat/src/renderer/inkling/mod.rs @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::collections::HashMap; + +use serde_json::{Map, Value, json}; +use thiserror_ext::AsReport as _; +use vllm_text::Prompt; +use vllm_text::tokenizer::{DynTokenizer, Tokenizer}; + +use super::{ChatRenderer, RenderedPrompt, request_template_kwargs}; +use crate::error::{Error, Result}; +use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool}; +use crate::{AssistantContentBlock, AssistantToolCall}; + +const MESSAGE_USER: &str = "<|message_user|>"; +const MESSAGE_MODEL: &str = "<|message_model|>"; +const MESSAGE_SYSTEM: &str = "<|message_system|>"; +const MESSAGE_TOOL: &str = "<|message_tool|>"; +const CONTENT_TEXT: &str = "<|content_text|>"; +const CONTENT_IMAGE: &str = "<|content_image|>"; +const CONTENT_MODEL_END_SAMPLING: &str = "<|content_model_end_sampling|>"; +const CONTENT_AUDIO_INPUT: &str = "<|content_audio_input|>"; +const CONTENT_THINKING: &str = "<|content_thinking|>"; +// Inkling renderer semantics use this slot for structured payloads such as tool +// declarations. +const CONTENT_XML: &str = "<|content_xml|>"; +const CONTENT_INVOKE_TOOL_JSON: &str = "<|content_invoke_tool_json|>"; +const END_MESSAGE: &str = "<|end_message|>"; +const AUDIO_END: &str = "<|audio_end|>"; +const MAX_REASONING_EFFORT: f64 = 0.99; + +/// Native Inkling renderer that emits token IDs directly. +#[derive(Clone)] +pub struct InklingChatRenderer { + tokenizer: DynTokenizer, + special: InklingSpecialTokenIds, +} + +#[derive(Debug, Clone, Copy)] +struct InklingSpecialTokenIds { + message_user: u32, + message_model: u32, + message_system: u32, + message_tool: u32, + content_text: u32, + content_image: u32, + content_model_end_sampling: u32, + content_audio_input: u32, + content_thinking: u32, + content_xml: u32, + content_invoke_tool_json: u32, + end_message: u32, + audio_end: u32, +} + +impl InklingSpecialTokenIds { + fn resolve(tokenizer: &dyn Tokenizer) -> Result { + Ok(Self { + message_user: resolve_special_token(tokenizer, MESSAGE_USER)?, + message_model: resolve_special_token(tokenizer, MESSAGE_MODEL)?, + message_system: resolve_special_token(tokenizer, MESSAGE_SYSTEM)?, + message_tool: resolve_special_token(tokenizer, MESSAGE_TOOL)?, + content_text: resolve_special_token(tokenizer, CONTENT_TEXT)?, + content_image: resolve_special_token(tokenizer, CONTENT_IMAGE)?, + content_model_end_sampling: resolve_special_token( + tokenizer, + CONTENT_MODEL_END_SAMPLING, + )?, + content_audio_input: resolve_special_token(tokenizer, CONTENT_AUDIO_INPUT)?, + content_thinking: resolve_special_token(tokenizer, CONTENT_THINKING)?, + content_xml: resolve_special_token(tokenizer, CONTENT_XML)?, + content_invoke_tool_json: resolve_special_token(tokenizer, CONTENT_INVOKE_TOOL_JSON)?, + end_message: resolve_special_token(tokenizer, END_MESSAGE)?, + audio_end: resolve_special_token(tokenizer, AUDIO_END)?, + }) + } +} + +impl InklingChatRenderer { + pub fn new(tokenizer: DynTokenizer) -> Result { + let special = InklingSpecialTokenIds::resolve(tokenizer.as_ref())?; + Ok(Self { tokenizer, special }) + } + + fn write_text_tokens(&self, out: &mut Vec, text: &str) -> Result<()> { + out.extend(self.tokenizer.encode(text, false)?); + Ok(()) + } + + fn write_message_start( + &self, + out: &mut Vec, + role_token_id: u32, + author_name: Option<&str>, + ) -> Result<()> { + out.push(role_token_id); + if let Some(author_name) = author_name + && !author_name.is_empty() + { + self.write_text_tokens(out, author_name)?; + } + Ok(()) + } + + fn write_text_block( + &self, + out: &mut Vec, + role_token_id: u32, + author_name: Option<&str>, + text: &str, + ) -> Result<()> { + self.write_message_start(out, role_token_id, author_name)?; + out.push(self.special.content_text); + self.write_text_tokens(out, text)?; + out.push(self.special.end_message); + Ok(()) + } + + /// Write one image block holding only the `<|content_image|>` marker. + /// Multimodal preprocessing later expands the marker into per-patch + /// image placeholder tokens once the patch count is known, mirroring the + /// Python `InklingMultiModalProcessor` marker-anchored prompt updates. + fn write_image_block(&self, out: &mut Vec, role_token_id: u32) { + out.push(role_token_id); + out.push(self.special.content_image); + out.push(self.special.end_message); + } + + /// Write one audio block holding the `<|content_audio_input|>` marker + /// followed by the `<|audio_end|>` terminator. Multimodal preprocessing + /// later expands the marker into per-frame audio placeholder tokens + /// (landing before `<|audio_end|>`) once the clip length is known. + fn write_audio_block(&self, out: &mut Vec, role_token_id: u32) { + out.push(role_token_id); + out.push(self.special.content_audio_input); + out.push(self.special.audio_end); + out.push(self.special.end_message); + } + + fn write_reasoning_block(&self, out: &mut Vec, text: &str) -> Result<()> { + if text.is_empty() { + return Ok(()); + } + out.push(self.special.message_model); + out.push(self.special.content_thinking); + self.write_text_tokens(out, text)?; + out.push(self.special.end_message); + Ok(()) + } + + fn write_reasoning_effort(&self, out: &mut Vec, effort: f64) -> Result<()> { + if !(0.0..=MAX_REASONING_EFFORT).contains(&effort) { + return Err(Error::ChatTemplate(format!( + "Inkling reasoning_effort must be in [0.0, 0.99], got {effort}" + ))); + } + let formatted = format!("{effort:.2}"); + let effort = formatted.trim_end_matches('0').trim_end_matches('.'); + let effort = if matches!(effort, "0" | "-0") { + "0.0" + } else { + effort + }; + self.write_text_block( + out, + self.special.message_system, + None, + &format!("Thinking effort level: {effort}"), + ) + } + + fn write_tool_declarations(&self, out: &mut Vec, tools: &[&ChatTool]) -> Result<()> { + if tools.is_empty() { + return Ok(()); + } + + let mut specs = Vec::with_capacity(tools.len()); + for tool in tools { + specs.push(json!({ + "description": tool.description.as_deref().unwrap_or(""), + "name": tool.name, + "parameters": sort_json(&tool.parameters), + "type": "function", + })); + } + let payload = compact_json(&sort_json(&Value::Array(specs)))?; + + self.write_message_start(out, self.special.message_system, Some("tool_declare"))?; + out.push(self.special.content_xml); + self.write_text_tokens(out, &payload)?; + out.push(self.special.end_message); + Ok(()) + } + + fn write_chat_content( + &self, + out: &mut Vec, + role_token_id: u32, + content: &ChatContent, + ) -> Result<()> { + match content { + ChatContent::Text(text) => { + if !text.is_empty() { + self.write_text_block(out, role_token_id, None, text)?; + } + } + ChatContent::Parts(parts) => { + for part in parts { + match part { + ChatContentPart::Text { text } => { + self.write_text_block(out, role_token_id, None, text)?; + } + ChatContentPart::ImageUrl { .. } => { + self.write_image_block(out, role_token_id); + } + ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => { + self.write_audio_block(out, role_token_id); + } + // Inkling has no video modality. + ChatContentPart::VideoUrl { .. } => { + return Err(Error::UnsupportedMultimodalContent("video_url")); + } + } + } + } + } + Ok(()) + } + + fn write_assistant_tool_call( + &self, + out: &mut Vec, + tool_call: &AssistantToolCall, + ) -> Result<()> { + let payload = tool_call_json(tool_call)?; + + self.write_message_start(out, self.special.message_model, Some(&tool_call.name))?; + out.push(self.special.content_invoke_tool_json); + self.write_text_tokens(out, &payload)?; + out.push(self.special.end_message); + Ok(()) + } + + fn write_assistant_content( + &self, + out: &mut Vec, + content: &[AssistantContentBlock], + tool_call_id_to_name: &mut HashMap, + ) -> Result<()> { + for block in content { + match block { + AssistantContentBlock::Reasoning { text } => { + self.write_reasoning_block(out, text)?; + } + AssistantContentBlock::Text { text } => { + if !text.is_empty() { + self.write_text_block(out, self.special.message_model, None, text)?; + } + } + AssistantContentBlock::ToolCall(tool_call) => { + if !tool_call.id.is_empty() { + tool_call_id_to_name.insert(tool_call.id.clone(), tool_call.name.clone()); + } + self.write_assistant_tool_call(out, tool_call)?; + } + } + } + out.push(self.special.content_model_end_sampling); + Ok(()) + } + + fn write_tool_response( + &self, + out: &mut Vec, + content: &ChatContent, + tool_call_id: &str, + tool_call_id_to_name: &HashMap, + ) -> Result<()> { + let text = content.try_flatten_to_text()?; + let tool_name = tool_call_id_to_name.get(tool_call_id).map(String::as_str).unwrap_or(""); + self.write_text_block(out, self.special.message_tool, Some(tool_name), &text) + } +} + +impl ChatRenderer for InklingChatRenderer { + fn render(&self, request: &ChatRequest) -> Result { + request.validate()?; + if request.chat_options.continue_final_message() { + return Err(Error::ChatTemplate( + "Inkling renderer does not support continue_final_message".to_string(), + )); + } + + let mut out = Vec::new(); + let mut tool_call_id_to_name = HashMap::new(); + let effective_template_kwargs = request_template_kwargs(request); + let tools = rendered_tools(request); + self.write_tool_declarations(&mut out, &tools)?; + let mut reasoning_effort = + resolve_reasoning_effort(effective_template_kwargs.get("reasoning_effort")); + + for message in &request.messages { + if !matches!( + message, + ChatMessage::System { .. } | ChatMessage::Developer { .. } + ) && let Some(effort) = reasoning_effort.take() + { + self.write_reasoning_effort(&mut out, effort)?; + } + + match message { + ChatMessage::System { content } => { + self.write_chat_content(&mut out, self.special.message_system, content)?; + } + ChatMessage::Developer { content, .. } => { + self.write_chat_content(&mut out, self.special.message_system, content)?; + } + ChatMessage::User { content } => { + self.write_chat_content(&mut out, self.special.message_user, content)?; + } + ChatMessage::Assistant { content } => { + self.write_assistant_content(&mut out, content, &mut tool_call_id_to_name)?; + } + ChatMessage::ToolResponse { + content, + tool_call_id, + } => { + self.write_tool_response( + &mut out, + content, + tool_call_id, + &tool_call_id_to_name, + )?; + } + } + } + + if let Some(effort) = reasoning_effort { + self.write_reasoning_effort(&mut out, effort)?; + } + + if request.chat_options.add_generation_prompt() { + out.push(self.special.message_model); + } + + Ok(RenderedPrompt { + prompt: Prompt::TokenIds(out), + effective_template_kwargs, + }) + } +} + +fn resolve_reasoning_effort(value: Option<&Value>) -> Option { + let Some(value) = value else { + return Some(0.9); + }; + match value { + Value::String(name) => match name.as_str() { + "none" => Some(0.0), + "minimal" => Some(0.1), + "low" => Some(0.2), + "medium" => Some(0.7), + "high" => Some(0.9), + "xhigh" | "max" => Some(0.99), + _ => None, + }, + Value::Number(number) => number.as_f64(), + _ => None, + } +} + +fn resolve_special_token(tokenizer: &dyn Tokenizer, token: &str) -> Result { + tokenizer.token_to_id(token).ok_or_else(|| { + Error::ChatTemplate(format!( + "Inkling tokenizer is missing special token `{token}`" + )) + }) +} + +fn rendered_tools(request: &ChatRequest) -> Vec<&ChatTool> { + if !request.tool_parsing_enabled() { + return Vec::new(); + } + + let mut tools = Vec::with_capacity(request.tools.len()); + tools.extend(request.tools.iter()); + for message in &request.messages { + if let ChatMessage::Developer { + tools: Some(local_tools), + .. + } = message + { + tools.extend(local_tools.iter()); + } + } + tools +} + +fn tool_call_json(tool_call: &AssistantToolCall) -> Result { + let name_json = serde_json::to_string(&tool_call.name) + .map_err(|error| Error::ChatTemplate(error.as_report().to_string()))?; + let arguments = if tool_call.arguments.trim().is_empty() { + Value::Object(Map::new()) + } else { + serde_json::from_str(&tool_call.arguments).map_err(|error| { + Error::ChatTemplate(format!( + "Inkling tool call arguments must decode to a JSON object: {error}" + )) + })? + }; + let Value::Object(_) = arguments else { + return Err(Error::ChatTemplate( + "Inkling tool call arguments must decode to a JSON object".to_string(), + )); + }; + let args_json = compact_json(&sort_json(&arguments))?; + Ok(format!("{{\"name\":{name_json},\"args\":{args_json}}}")) +} + +fn compact_json(value: &Value) -> Result { + serde_json::to_string(value).map_err(|error| Error::ChatTemplate(error.as_report().to_string())) +} + +fn sort_json(value: &Value) -> Value { + match value { + Value::Array(items) => Value::Array(items.iter().map(sort_json).collect()), + Value::Object(map) => { + let mut sorted = Map::new(); + let mut keys = map.keys().collect::>(); + keys.sort(); + for key in keys { + sorted.insert(key.clone(), sort_json(&map[key])); + } + Value::Object(sorted) + } + _ => value.clone(), + } +} + +#[cfg(test)] +mod tests; diff --git a/rust/src/chat/src/renderer/inkling/tests.rs b/rust/src/chat/src/renderer/inkling/tests.rs new file mode 100644 index 0000000000..d0d55e1be2 --- /dev/null +++ b/rust/src/chat/src/renderer/inkling/tests.rs @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::path::PathBuf; +use std::sync::Arc; + +use expect_test::{ExpectFile, expect_file}; +use serde_json::json; +use thiserror_ext::AsReport; +use vllm_text::tokenizer::Tokenizer; + +use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request}; + +use super::{ + AUDIO_END, CONTENT_AUDIO_INPUT, CONTENT_IMAGE, CONTENT_INVOKE_TOOL_JSON, CONTENT_TEXT, + CONTENT_THINKING, CONTENT_XML, END_MESSAGE, InklingChatRenderer, MESSAGE_MODEL, MESSAGE_SYSTEM, + MESSAGE_TOOL, MESSAGE_USER, +}; +use crate::event::{AssistantContentBlock, AssistantToolCall}; +use crate::request::{ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort}; +use crate::{ChatRenderer, Error}; + +struct FixtureTokenizer; + +impl Tokenizer for FixtureTokenizer { + fn encode( + &self, + text: &str, + _add_special_tokens: bool, + ) -> vllm_text::tokenizer::Result> { + Ok(text.bytes().map(u32::from).collect()) + } + + fn decode( + &self, + token_ids: &[u32], + _skip_special_tokens: bool, + ) -> vllm_text::tokenizer::Result { + Ok(token_ids + .iter() + .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) + .collect()) + } + + fn token_to_id(&self, token: &str) -> Option { + match token { + "<|message_user|>" => Some(200000), + "<|message_model|>" => Some(200001), + "<|message_system|>" => Some(200002), + "<|message_tool|>" => Some(200003), + "<|content_text|>" => Some(200004), + "<|content_image|>" => Some(200005), + "<|content_model_end_sampling|>" => Some(200006), + "<|content_thinking|>" => Some(200008), + "<|end_message|>" => Some(200010), + "<|content_audio_input|>" => Some(200020), + CONTENT_XML => Some(200024), + "<|audio_end|>" => Some(200043), + "<|content_invoke_tool_json|>" => Some(200049), + _ => None, + } + } + + fn id_to_token(&self, id: u32) -> Option { + let token = match id { + 200000 => "<|message_user|>", + 200001 => "<|message_model|>", + 200002 => "<|message_system|>", + 200003 => "<|message_tool|>", + 200004 => "<|content_text|>", + 200005 => "<|content_image|>", + 200006 => "<|content_model_end_sampling|>", + 200008 => "<|content_thinking|>", + 200010 => "<|end_message|>", + 200020 => "<|content_audio_input|>", + 200024 => CONTENT_XML, + 200043 => "<|audio_end|>", + 200049 => "<|content_invoke_tool_json|>", + _ => return None, + }; + Some(token.to_string()) + } +} + +fn renderer() -> InklingChatRenderer { + InklingChatRenderer::new(Arc::new(FixtureTokenizer)).unwrap() +} + +fn render_token_ids(request: &ChatRequest) -> Vec { + renderer() + .render(request) + .unwrap() + .prompt + .into_token_ids() + .expect("Inkling renderer returns token IDs") +} + +fn fixture_request(name: &str) -> ChatRequest { + fixture_chat_request(&fixture_path(name), inkling_fixture_options()) +} + +fn inkling_fixture_options() -> FixtureRequestOptions { + FixtureRequestOptions { + enable_thinking: false, + no_generation_prompt_when_last_assistant: false, + } +} + +fn fixture_path(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/renderer/inkling") + .join("fixtures") + .join(name) +} + +fn assert_fixture(input_name: &str, expected: ExpectFile) { + let request = fixture_request(input_name); + let rendered = format!("{}\n", render_symbolic_tokens(&render_token_ids(&request))); + + expected.assert_eq(&rendered); +} + +fn render_symbolic_tokens(token_ids: &[u32]) -> String { + let mut out = String::new(); + let mut text_bytes = Vec::new(); + for token_id in token_ids { + if let Some(marker) = symbolic_tokens() + .iter() + .find_map(|(marker, marker_id)| (token_id == marker_id).then_some(*marker)) + { + flush_text_bytes(&mut out, &mut text_bytes); + out.push_str(marker); + continue; + } + + let byte = u8::try_from(*token_id) + .unwrap_or_else(|_| panic!("unexpected non-byte fixture text token {token_id}")); + text_bytes.push(byte); + } + flush_text_bytes(&mut out, &mut text_bytes); + out +} + +fn flush_text_bytes(out: &mut String, text_bytes: &mut Vec) { + if text_bytes.is_empty() { + return; + } + out.push_str(std::str::from_utf8(text_bytes).unwrap()); + text_bytes.clear(); +} + +fn symbolic_tokens() -> [(&'static str, u32); 13] { + [ + (MESSAGE_USER, 200000), + (MESSAGE_MODEL, 200001), + (MESSAGE_SYSTEM, 200002), + (MESSAGE_TOOL, 200003), + (CONTENT_TEXT, 200004), + (CONTENT_IMAGE, 200005), + (super::CONTENT_MODEL_END_SAMPLING, 200006), + (CONTENT_THINKING, 200008), + (END_MESSAGE, 200010), + (CONTENT_AUDIO_INPUT, 200020), + (CONTENT_XML, 200024), + (AUDIO_END, 200043), + (CONTENT_INVOKE_TOOL_JSON, 200049), + ] +} + +#[test] +fn renders_text_and_image_as_inkling_tokens() { + assert_fixture( + "text_image_input.json", + expect_file!["fixtures/text_image_output.txt"], + ); +} + +#[test] +fn renders_audio_marker_only_blocks() { + assert_fixture( + "text_audio_input.json", + expect_file!["fixtures/text_audio_output.txt"], + ); +} + +#[test] +fn renders_reasoning_text_tool_call_and_tool_response() { + assert_fixture( + "tool_round_trip_input.json", + expect_file!["fixtures/tool_round_trip_output.txt"], + ); +} + +#[test] +fn renders_request_and_developer_tools_as_tool_declare() { + assert_fixture( + "tool_declare_input.json", + expect_file!["fixtures/tool_declare_output.txt"], + ); +} + +#[test] +fn renders_named_reasoning_effort_after_tool_declarations() { + for (effort, expected) in [ + (ReasoningEffort::None, "0.0"), + (ReasoningEffort::Minimal, "0.1"), + (ReasoningEffort::Low, "0.2"), + (ReasoningEffort::Medium, "0.7"), + (ReasoningEffort::High, "0.9"), + (ReasoningEffort::XHigh, "0.99"), + (ReasoningEffort::Max, "0.99"), + ] { + let mut request = fixture_request("tool_declare_input.json"); + request.chat_options.reasoning_effort = Some(effort); + + let rendered = render_symbolic_tokens(&render_token_ids(&request)); + let tool_end = rendered.find("<|end_message|>").unwrap(); + let effort_block = format!( + "<|message_system|><|content_text|>Thinking effort level: \ + {expected}<|end_message|>" + ); + let effort_start = rendered.find(&effort_block).unwrap(); + let system_end = rendered.find("rules<|end_message|>").unwrap(); + + assert!(effort_start > tool_end); + assert!(effort_start > system_end); + } +} + +#[test] +fn emits_one_reasoning_effort_for_multi_turn_conversation() { + let request = ChatRequest { + messages: vec![ + ChatMessage::system("rules"), + ChatMessage::user("user1"), + ChatMessage::assistant_text("assistant1"), + ChatMessage::user("user2"), + ], + ..ChatRequest::for_test() + }; + + let rendered = render_symbolic_tokens(&render_token_ids(&request)); + let effort = "<|message_system|><|content_text|>Thinking effort level: 0.9\ + <|end_message|>"; + + assert_eq!(rendered.matches(effort).count(), 1); + assert!(rendered.find("rules").unwrap() < rendered.find(effort).unwrap()); + assert!(rendered.find(effort).unwrap() < rendered.find("user1").unwrap()); + assert!(rendered.find("user1").unwrap() < rendered.find("assistant1").unwrap()); + assert!(rendered.find("assistant1").unwrap() < rendered.find("user2").unwrap()); +} + +#[test] +fn canonicalizes_numeric_zero_reasoning_effort() { + for value in [0.0, -0.0] { + let mut request = ChatRequest::for_test(); + request + .chat_options + .template_kwargs + .insert("reasoning_effort".to_string(), json!(value)); + + assert!( + render_symbolic_tokens(&render_token_ids(&request)).starts_with( + "<|message_system|><|content_text|>Thinking effort level: 0.0\ + <|end_message|>" + ) + ); + } +} + +#[test] +fn defaults_reasoning_effort_to_high() { + let request = ChatRequest::for_test(); + + assert!( + render_symbolic_tokens(&render_token_ids(&request)).starts_with( + "<|message_system|><|content_text|>Thinking effort level: 0.9\ + <|end_message|>" + ) + ); +} + +#[test] +fn renders_numeric_reasoning_effort_template_kwarg() { + let mut request = ChatRequest::for_test(); + request + .chat_options + .template_kwargs + .insert("reasoning_effort".to_string(), json!(0.8)); + + assert_eq!( + render_symbolic_tokens(&render_token_ids(&request)), + "<|message_system|><|content_text|>Thinking effort level: 0.8\ + <|end_message|><|message_user|><|content_text|>test\ + <|end_message|><|message_model|>" + ); +} + +#[test] +fn ignores_unsupported_reasoning_effort_values() { + for value in [json!(true), json!("invalid"), json!(null)] { + let mut request = ChatRequest::for_test(); + request + .chat_options + .template_kwargs + .insert("reasoning_effort".to_string(), value); + + assert_eq!( + render_symbolic_tokens(&render_token_ids(&request)), + "<|message_user|><|content_text|>test<|end_message|><|message_model|>" + ); + } +} + +#[test] +fn rejects_out_of_range_reasoning_effort() { + for value in [0.990_000_1, 1.0, 1.5, -0.1] { + let mut request = ChatRequest::for_test(); + request + .chat_options + .template_kwargs + .insert("reasoning_effort".to_string(), json!(value)); + + let error = renderer().render(&request).unwrap_err(); + assert!(error.as_report().to_string().contains("must be in [0.0, 0.99]")); + } +} + +#[test] +fn rejects_continue_final_message() { + let request = ChatRequest { + messages: vec![ChatMessage::assistant_text("partial")], + chat_options: crate::ChatOptions { + generation_prompt_mode: GenerationPromptMode::ContinueFinalAssistant, + ..Default::default() + }, + ..ChatRequest::for_test() + }; + + let error = renderer().render(&request).unwrap_err(); + assert!(matches!(error, Error::ChatTemplate(_))); + assert!(error.as_report().to_string().contains("continue_final_message")); +} + +#[test] +fn renders_developer_messages_as_system() { + let request = ChatRequest { + messages: vec![ChatMessage::developer("rules", None)], + ..ChatRequest::for_test() + }; + + assert_eq!( + render_symbolic_tokens(&render_token_ids(&request)), + "<|message_system|><|content_text|>rules<|end_message|>\ + <|message_system|><|content_text|>Thinking effort level: 0.9\ + <|end_message|><|message_model|>" + ); +} + +#[test] +fn rejects_non_object_tool_call_arguments() { + let request = ChatRequest { + messages: vec![ChatMessage::assistant_blocks(vec![ + AssistantContentBlock::ToolCall(AssistantToolCall { + id: "call_1".to_string(), + name: "get_weather".to_string(), + arguments: "[]".to_string(), + }), + ])], + ..ChatRequest::for_test() + }; + + let error = renderer().render(&request).unwrap_err(); + assert!(error.as_report().to_string().contains("JSON object")); +} diff --git a/rust/src/chat/src/renderer/mod.rs b/rust/src/chat/src/renderer/mod.rs index 639d6df30e..4d9c1581a0 100644 --- a/rust/src/chat/src/renderer/mod.rs +++ b/rust/src/chat/src/renderer/mod.rs @@ -14,6 +14,7 @@ pub mod deepseek_v32; pub mod deepseek_v4; pub mod harmony; pub mod hf; +mod inkling; mod selection; #[cfg(test)] mod test_utils; @@ -21,6 +22,7 @@ mod test_utils; pub use deepseek_v4::DeepSeekV4ChatRenderer; pub use deepseek_v32::DeepSeekV32ChatRenderer; pub use harmony::HarmonyChatRenderer; +pub use inkling::InklingChatRenderer; pub use selection::RendererSelection; /// Rendered chat prompt submitted to the text backend. diff --git a/rust/src/chat/src/renderer/selection.rs b/rust/src/chat/src/renderer/selection.rs index 508bf8d17c..711c0d87dd 100644 --- a/rust/src/chat/src/renderer/selection.rs +++ b/rust/src/chat/src/renderer/selection.rs @@ -24,6 +24,8 @@ pub enum RendererSelection { DeepSeekV4, /// Force the GPT-OSS Harmony renderer. Harmony, + /// Force the Inkling native token renderer. + Inkling, } impl RendererSelection { @@ -33,6 +35,8 @@ impl RendererSelection { pub const GPT_OSS_MODEL_TYPE: &str = "gpt_oss"; pub const HARMONY_LITERAL: &str = "harmony"; pub const HF_LITERAL: &str = "hf"; + pub const INKLING_LITERAL: &str = "inkling"; + pub const INKLING_MODEL_TYPE: &str = "inkling_mm_model"; /// Resolve the renderer selection using the given model type string, if /// it's `Auto`. @@ -42,6 +46,7 @@ impl RendererSelection { Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32, Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4, Self::GPT_OSS_MODEL_TYPE => Self::Harmony, + Self::INKLING_MODEL_TYPE => Self::Inkling, _ => Self::Hf, }, selection => selection, @@ -63,6 +68,8 @@ impl FromStr for RendererSelection { Ok(Self::DeepSeekV4) } else if value.eq_ignore_ascii_case(Self::HARMONY_LITERAL) { Ok(Self::Harmony) + } else if value.eq_ignore_ascii_case(Self::INKLING_LITERAL) { + Ok(Self::Inkling) } else { Err(format!( "unknown renderer `{value}` (expected one of: {})", @@ -80,6 +87,7 @@ impl fmt::Display for RendererSelection { Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL), Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL), Self::Harmony => f.write_str(Self::HARMONY_LITERAL), + Self::Inkling => f.write_str(Self::INKLING_LITERAL), } } } @@ -106,7 +114,7 @@ mod tests { fn renderer_selection_expected_error_message() { let err = RendererSelection::from_str("unknown").unwrap_err(); expect_test::expect![ - "unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)" + "unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling)" ] .assert_eq(&err); } diff --git a/rust/src/chat/src/renderer/test_utils.rs b/rust/src/chat/src/renderer/test_utils.rs index a12531150c..08eeca8176 100644 --- a/rust/src/chat/src/renderer/test_utils.rs +++ b/rust/src/chat/src/renderer/test_utils.rs @@ -103,6 +103,8 @@ pub(crate) enum FixtureContent { pub(crate) enum FixtureContentPart { Text { text: String }, ImageUrl { image_url: String }, + InputAudio { data: String }, + AudioUrl { audio_url: String }, } #[derive(Debug, Deserialize)] @@ -226,6 +228,15 @@ fn to_chat_content(content: FixtureContent) -> ChatContent { FixtureContentPart::ImageUrl { image_url } => { ChatContentPart::image_url(image_url) } + FixtureContentPart::InputAudio { data } => ChatContentPart::InputAudio { + data, + format: None, + uuid: None, + }, + FixtureContentPart::AudioUrl { audio_url } => ChatContentPart::AudioUrl { + audio_url, + uuid: None, + }, }) .collect(), ), diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index 58a3adfc4e..e06bf6e5da 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -263,19 +263,32 @@ impl RoundtripCase { sort_json_keys: false, } } + + /// Inkling typed content blocks with native token-id rendering. + fn inkling() -> Self { + Self { + model_id: "thinkingmachines/Inkling", + assistant_stop_suffix: "", + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::Auto, + thinking_behavior: ThinkingBehavior::Always { value: true }, + json_fmt: compact_json_fmt(), + sort_json_keys: true, + } + } } macro_rules! roundtrip_tests { - ($($case:ident => [$($(#[$fixture_attr:meta])* $fixture:ident),* $(,)?]),+ $(,)?) => { + ($($case:ident => $(#[$case_attr:meta])* [$($fixture:ident),* $(,)?]),+ $(,)?) => { paste::paste! { $( #[tokio::test] #[file_serial([])] + $(#[$case_attr])* async fn []() -> Result<()> { let case = RoundtripCase::$case(); let backends = load_roundtrip_backends(&case).await?; $( - $(#[$fixture_attr])* [](&case, &backends).await?; )* Ok(()) @@ -300,6 +313,7 @@ roundtrip_tests! { gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call + inkling => [reasoning_and_content, tool_call_mix], } /// Run the fixed reasoning+content fixture for one model/parser case. diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index f3af2a18fd..ee0d3ebf04 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -646,7 +646,7 @@ fn serve_args_reject_unknown_renderer_value() { .unwrap_err(); expect![[r#" - error: invalid value 'definitely_missing' for '--tokenizer-mode ': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony) + error: invalid value 'definitely_missing' for '--tokenizer-mode ': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling) For more information, try '--help'. "#]] diff --git a/rust/src/parser/src/tool/json/mod.rs b/rust/src/parser/src/tool/json/mod.rs index b707bff6fd..d545ece23e 100644 --- a/rust/src/parser/src/tool/json/mod.rs +++ b/rust/src/parser/src/tool/json/mod.rs @@ -31,24 +31,24 @@ use super::utils::{ }; use super::{Result, ToolCallDelta, ToolParserOutput}; -type JsonToolInput<'i> = Partial<&'i str>; +pub(crate) type JsonToolInput<'i> = Partial<&'i str>; #[derive(Debug, Clone, Copy)] -struct JsonToolCallConfig { - parser_name: &'static str, - start_marker: &'static str, - end_marker: &'static str, - marker_whitespace: JsonToolCallWhitespace, - delimiter: Option<&'static str>, - name_key: &'static str, +pub(crate) struct JsonToolCallConfig { + pub parser_name: &'static str, + pub start_marker: &'static str, + pub end_marker: &'static str, + pub marker_whitespace: JsonToolCallWhitespace, + pub delimiter: Option<&'static str>, + pub name_key: &'static str, /// Candidate JSON keys naming the arguments payload, tried in order. /// Most parsers use a single key like `["arguments"]`, but some accept /// multiple (e.g. InternLM2 accepts `parameters` or `arguments`). - arguments_key: &'static [&'static str], + pub arguments_key: &'static [&'static str], } #[derive(Debug, Clone, Copy)] -enum JsonToolCallWhitespace { +pub(crate) enum JsonToolCallWhitespace { Optional, Exact(&'static str), } @@ -61,7 +61,7 @@ enum JsonToolCallMode { } #[derive(Debug, Clone, PartialEq, Eq)] -enum JsonToolCallEvent { +pub(crate) enum JsonToolCallEvent { Text { len: usize }, ToolCallStart, ToolCallHeader { function_name: String }, @@ -220,7 +220,7 @@ fn tool_call_start_event( /// Parse a marker-wrapped JSON tool-call header before the raw arguments /// payload. -fn tool_call_header_event( +pub(crate) fn tool_call_header_event( input: &mut JsonToolInput<'_>, config: JsonToolCallConfig, ) -> ModalResult { diff --git a/rust/src/parser/src/tool/mod.rs b/rust/src/parser/src/tool/mod.rs index d0e88e1936..0ee72093b4 100644 --- a/rust/src/parser/src/tool/mod.rs +++ b/rust/src/parser/src/tool/mod.rs @@ -9,7 +9,7 @@ mod deepseek_dsml; pub(crate) mod deepseek_json; mod glm_xml; mod hy_v3; -mod json; +pub(crate) mod json; mod kimi_k2; mod minimax_m2; mod minimax_m3; diff --git a/rust/src/parser/src/unified/gemma4.rs b/rust/src/parser/src/unified/gemma4.rs index bc5925189e..8577a3fc28 100644 --- a/rust/src/parser/src/unified/gemma4.rs +++ b/rust/src/parser/src/unified/gemma4.rs @@ -10,7 +10,7 @@ use winnow::prelude::*; use winnow::stream::{Partial, Stream}; use winnow::token::{literal, take_till, take_until}; -use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput}; +use super::{Result, UnifiedParser, UnifiedParserOutput, token_id}; use crate::reasoning::last_reasoning_boundary; use crate::tool::{Tool, ToolCallDelta}; use crate::unified::parsing_failed; @@ -79,17 +79,8 @@ pub struct Gemma4UnifiedParser { impl Gemma4UnifiedParser { /// Create a Gemma4 parser. pub fn new(_tools: &[Tool], tokenizer: DynTokenizer) -> Result { - let channel_start_token_id = tokenizer.token_to_id(CHANNEL_START).ok_or_else(|| { - UnifiedParserError::MissingToken { - token: CHANNEL_START.to_string(), - } - })?; - let channel_end_token_id = - tokenizer - .token_to_id(CHANNEL_END) - .ok_or_else(|| UnifiedParserError::MissingToken { - token: CHANNEL_END.to_string(), - })?; + let channel_start_token_id = token_id(tokenizer.as_ref(), CHANNEL_START)?; + let channel_end_token_id = token_id(tokenizer.as_ref(), CHANNEL_END)?; Ok(Self { buffer: String::new(), @@ -524,10 +515,10 @@ mod tests { use super::{ CHANNEL_END, CHANNEL_START, Gemma4UnifiedParser, ToolCallDelta, UnifiedParser, - UnifiedParserError, UnifiedParserOutput, gemma4_array_content, parse_gemma4_args, + UnifiedParserOutput, gemma4_array_content, parse_gemma4_args, }; use crate::tool::Tool; - use crate::unified::{UnifiedParserEvent, parsing_failed}; + use crate::unified::{UnifiedParserError, UnifiedParserEvent, parsing_failed}; const CHANNEL_START_ID: u32 = 256; const CHANNEL_END_ID: u32 = 257; diff --git a/rust/src/parser/src/unified/inkling.rs b/rust/src/parser/src/unified/inkling.rs new file mode 100644 index 0000000000..eb78321d12 --- /dev/null +++ b/rust/src/parser/src/unified/inkling.rs @@ -0,0 +1,768 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use winnow::ascii::multispace0 as ws0; +use winnow::combinator::{alt, seq}; +use winnow::error::ModalResult; +use winnow::prelude::*; +use winnow::stream::Partial; +use winnow::token::literal; + +use vllm_tokenizer::DynTokenizer; + +use super::{Result, UnifiedParser, UnifiedParserOutput, token_id}; +use crate::tool::json::{ + JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput, + tool_call_header_event, +}; +use crate::tool::{Tool, ToolCallDelta}; +use crate::unified::parsing_failed; +use crate::utils::{ + JsonObjectScanState, parse_buffered_event, safe_text_len_mul, take_json_object, +}; + +const CONTENT_TEXT: &str = "<|content_text|>"; +const CONTENT_THINKING: &str = "<|content_thinking|>"; +const CONTENT_INVOKE_TOOL_JSON: &str = "<|content_invoke_tool_json|>"; +const CONTENT_INVOKE_TOOL_TEXT: &str = "<|content_invoke_tool_text|>"; +const CONTENT_MODEL_END_SAMPLING: &str = "<|content_model_end_sampling|>"; +const CONTENT_TOOL_ERROR: &str = "<|content_tool_error|>"; +const MESSAGE_MODEL: &str = "<|message_model|>"; +const END_MESSAGE: &str = "<|end_message|>"; + +const IDLE_MARKERS: &[&str] = &[ + MESSAGE_MODEL, + CONTENT_TEXT, + CONTENT_THINKING, + CONTENT_INVOKE_TOOL_JSON, + CONTENT_INVOKE_TOOL_TEXT, + CONTENT_MODEL_END_SAMPLING, + CONTENT_TOOL_ERROR, + END_MESSAGE, +]; +const BLOCK_END_MARKERS: &[&str] = &[END_MESSAGE, CONTENT_MODEL_END_SAMPLING]; + +const INKLING_TOOL_CONFIG: JsonToolCallConfig = JsonToolCallConfig { + parser_name: "Inkling", + start_marker: CONTENT_INVOKE_TOOL_JSON, + end_marker: END_MESSAGE, + marker_whitespace: JsonToolCallWhitespace::Optional, + delimiter: None, + name_key: "name", + arguments_key: &["args"], +}; + +type InklingInput<'i> = Partial<&'i str>; + +#[derive(Debug, Clone, PartialEq, Eq)] +enum InklingEvent { + Text { len: usize }, + Reasoning { len: usize }, + TextStart, + ReasoningStart, + MessageStart, + Header, + ToolJsonStart, + ToolJsonHeader { name: String }, + ToolJsonArgs { len: usize, complete: bool }, + BlockEnd, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +enum InklingMode { + #[default] + Idle, + MessageHeader, + Text, + Reasoning, + ToolJsonHeader, + ToolJsonArgs { + json_scan: JsonObjectScanState, + }, + ToolJsonClose, +} + +/// Unified parser for Inkling typed content blocks. +pub struct InklingUnifiedParser { + buffer: String, + mode: InklingMode, + emitted_tool_count: usize, + active_tool_index: Option, + tokenizer: DynTokenizer, + message_model_token_id: u32, + content_text_token_id: u32, + content_thinking_token_id: u32, +} + +impl InklingUnifiedParser { + /// Create a Inkling parser. + pub fn new(_tools: &[Tool], tokenizer: DynTokenizer) -> Result { + let message_model_token_id = token_id(tokenizer.as_ref(), MESSAGE_MODEL)?; + let content_text_token_id = token_id(tokenizer.as_ref(), CONTENT_TEXT)?; + let content_thinking_token_id = token_id(tokenizer.as_ref(), CONTENT_THINKING)?; + + Ok(Self { + buffer: String::new(), + mode: InklingMode::Idle, + emitted_tool_count: 0, + active_tool_index: None, + tokenizer, + message_model_token_id, + content_text_token_id, + content_thinking_token_id, + }) + } + + fn initialize_mode(&mut self, prompt_token_ids: &[u32]) { + self.mode = InklingMode::Idle; + for token_id in prompt_token_ids.iter().rev().copied() { + if token_id == self.message_model_token_id { + self.mode = InklingMode::MessageHeader; + return; + } + if token_id == self.content_thinking_token_id { + self.mode = InklingMode::Reasoning; + return; + } + if token_id == self.content_text_token_id { + self.mode = InklingMode::Text; + return; + } + if self.tokenizer.is_special_id(token_id) { + return; + } + } + } + + fn apply_event(&mut self, event: InklingEvent, output: &mut UnifiedParserOutput) -> Result<()> { + match event { + InklingEvent::Text { len } => output.push_text(self.buffer[..len].to_string()), + InklingEvent::Reasoning { len } => { + output.push_reasoning(self.buffer[..len].to_string()); + } + InklingEvent::MessageStart => self.mode = InklingMode::MessageHeader, + InklingEvent::Header => {} + InklingEvent::TextStart => self.mode = InklingMode::Text, + InklingEvent::ReasoningStart => self.mode = InklingMode::Reasoning, + InklingEvent::ToolJsonStart => self.mode = InklingMode::ToolJsonHeader, + InklingEvent::ToolJsonHeader { name } => { + let tool_index = self.emitted_tool_count; + self.emitted_tool_count += 1; + self.active_tool_index = Some(tool_index); + self.mode = InklingMode::ToolJsonArgs { + json_scan: JsonObjectScanState::default(), + }; + output.push_call(ToolCallDelta { + tool_index, + name: Some(name), + arguments: String::new(), + }); + } + InklingEvent::ToolJsonArgs { len, complete } => { + let Some(tool_index) = self.active_tool_index else { + return Err(parsing_failed!( + "Inkling arguments without an active tool call" + )); + }; + output.push_call(ToolCallDelta { + tool_index, + name: None, + arguments: self.buffer[..len].to_string(), + }); + if complete { + self.mode = InklingMode::ToolJsonClose; + } + } + InklingEvent::BlockEnd => { + self.mode = InklingMode::Idle; + self.active_tool_index = None; + } + } + Ok(()) + } + + fn reset(&mut self) -> String { + self.mode = InklingMode::Idle; + self.active_tool_index = None; + self.emitted_tool_count = 0; + std::mem::take(&mut self.buffer) + } +} + +impl UnifiedParser for InklingUnifiedParser { + fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result> + where + Self: Sized + 'static, + { + Self::new(tools, tokenizer).map(|parser| Box::new(parser) as Box) + } + + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { + self.buffer.clear(); + self.emitted_tool_count = 0; + self.active_tool_index = None; + self.initialize_mode(prompt_token_ids); + Ok(()) + } + + fn preserve_special_tokens(&self) -> bool { + true + } + + fn parse_into(&mut self, chunk: &str, output: &mut UnifiedParserOutput) -> Result<()> { + self.buffer.push_str(chunk); + + while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| { + parse_next_inkling_event(input, &mut self.mode) + })? { + self.apply_event(event, output)?; + self.buffer.drain(..consumed_len); + } + + Ok(()) + } + + fn finish(&mut self) -> Result { + let mut output = UnifiedParserOutput::default(); + + match &self.mode { + InklingMode::Idle | InklingMode::Text => { + output.push_text(std::mem::take(&mut self.buffer)) + } + InklingMode::MessageHeader => self.buffer.clear(), + InklingMode::Reasoning => output.push_reasoning(std::mem::take(&mut self.buffer)), + InklingMode::ToolJsonHeader + | InklingMode::ToolJsonArgs { .. } + | InklingMode::ToolJsonClose => { + return Err(parsing_failed!("incomplete Inkling tool call")); + } + } + + let _ = self.reset(); + Ok(output) + } + + fn reset(&mut self) -> String { + InklingUnifiedParser::reset(self) + } +} + +/// Parse one Inkling event from buffered streaming input. +fn parse_next_inkling_event( + input: &mut InklingInput<'_>, + mode: &mut InklingMode, +) -> ModalResult { + match mode { + InklingMode::Idle => parse_idle_event(input), + InklingMode::MessageHeader => parse_message_header_event(input), + InklingMode::Text => parse_text_event(input), + InklingMode::Reasoning => parse_reasoning_event(input), + InklingMode::ToolJsonHeader => parse_tool_json_header_event(input), + InklingMode::ToolJsonArgs { json_scan } => parse_tool_json_args_event(input, json_scan), + InklingMode::ToolJsonClose => parse_tool_json_close_event(input), + } +} + +/// Parse an event while waiting for a Inkling content kind. +fn parse_idle_event(input: &mut InklingInput<'_>) -> ModalResult { + alt(( + message_start_event, + reasoning_start_event, + text_start_event, + tool_json_start_event, + raw_text_start_event, + block_end_event, + safe_idle_text_event, + )) + .parse_next(input) +} + +/// Parse a Inkling model-authored message start marker. +fn message_start_event(input: &mut InklingInput<'_>) -> ModalResult { + literal(MESSAGE_MODEL).value(InklingEvent::MessageStart).parse_next(input) +} + +/// Parse an event while waiting for an Inkling message content kind. +fn parse_message_header_event(input: &mut InklingInput<'_>) -> ModalResult { + alt(( + reasoning_start_event, + text_start_event, + tool_json_start_event, + raw_text_start_event, + block_end_event, + safe_header_event, + )) + .parse_next(input) +} + +/// Parse an event inside a Inkling text block. +fn parse_text_event(input: &mut InklingInput<'_>) -> ModalResult { + alt((block_end_event, safe_text_event)).parse_next(input) +} + +/// Parse an event inside a Inkling reasoning block. +fn parse_reasoning_event(input: &mut InklingInput<'_>) -> ModalResult { + alt((block_end_event, safe_reasoning_event)).parse_next(input) +} + +/// Parse a Inkling text start marker. +fn text_start_event(input: &mut InklingInput<'_>) -> ModalResult { + literal(CONTENT_TEXT).value(InklingEvent::TextStart).parse_next(input) +} + +/// Parse a Inkling reasoning start marker. +fn reasoning_start_event(input: &mut InklingInput<'_>) -> ModalResult { + literal(CONTENT_THINKING).value(InklingEvent::ReasoningStart).parse_next(input) +} + +/// Parse a Inkling JSON tool-call start marker. +fn tool_json_start_event(input: &mut InklingInput<'_>) -> ModalResult { + literal(CONTENT_INVOKE_TOOL_JSON) + .value(InklingEvent::ToolJsonStart) + .parse_next(input) +} + +/// Parse a Inkling content kind treated as visible text. +fn raw_text_start_event(input: &mut InklingInput<'_>) -> ModalResult { + alt(( + literal(CONTENT_INVOKE_TOOL_TEXT), + literal(CONTENT_TOOL_ERROR), + )) + .value(InklingEvent::TextStart) + .parse_next(input) +} + +/// Parse a Inkling block end marker. +fn block_end_event(input: &mut InklingInput<'_>) -> ModalResult { + alt((literal(END_MESSAGE), literal(CONTENT_MODEL_END_SAMPLING))) + .value(InklingEvent::BlockEnd) + .parse_next(input) +} + +/// Parse safe text while waiting for the next Inkling marker. +fn safe_idle_text_event(input: &mut InklingInput<'_>) -> ModalResult { + safe_text_len_mul(input, IDLE_MARKERS).map(|len| InklingEvent::Text { len }) +} + +/// Parse safe header text before the next Inkling marker. +fn safe_header_event(input: &mut InklingInput<'_>) -> ModalResult { + safe_text_len_mul(input, IDLE_MARKERS).map(|_| InklingEvent::Header) +} + +/// Parse safe text before the end of a Inkling text block. +fn safe_text_event(input: &mut InklingInput<'_>) -> ModalResult { + safe_text_len_mul(input, BLOCK_END_MARKERS).map(|len| InklingEvent::Text { len }) +} + +/// Parse safe reasoning before the end of a Inkling reasoning block. +fn safe_reasoning_event(input: &mut InklingInput<'_>) -> ModalResult { + safe_text_len_mul(input, BLOCK_END_MARKERS).map(|len| InklingEvent::Reasoning { len }) +} + +/// Parse a Inkling JSON tool-call header. +fn parse_tool_json_header_event(input: &mut InklingInput<'_>) -> ModalResult { + match tool_call_header_event(input, INKLING_TOOL_CONFIG)? { + JsonToolCallEvent::ToolCallHeader { function_name } => Ok(InklingEvent::ToolJsonHeader { + name: function_name, + }), + _ => unreachable!("tool_call_header_event only emits ToolCallHeader"), + } +} + +/// Parse raw Inkling JSON tool-call argument bytes. +fn parse_tool_json_args_event( + input: &mut JsonToolInput<'_>, + json_scan: &mut JsonObjectScanState, +) -> ModalResult { + let len = take_json_object(input, json_scan)?; + Ok(InklingEvent::ToolJsonArgs { + len, + complete: json_scan.complete(), + }) +} + +/// Parse the close of a Inkling JSON tool-call block. +fn parse_tool_json_close_event(input: &mut InklingInput<'_>) -> ModalResult { + seq!( + _: ws0, + _: literal("}"), + _: ws0, + _: literal(END_MESSAGE), + ) + .value(InklingEvent::BlockEnd) + .parse_next(input) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::{CONTENT_TEXT, CONTENT_THINKING, InklingUnifiedParser, MESSAGE_MODEL}; + use crate::tool::Tool; + use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput}; + use thiserror_ext::AsReport; + use vllm_tokenizer::Tokenizer; + + struct FakeTokenizer; + + impl Tokenizer for FakeTokenizer { + fn encode( + &self, + text: &str, + _add_special_tokens: bool, + ) -> vllm_tokenizer::Result> { + Ok(text.chars().map(u32::from).collect()) + } + + fn decode( + &self, + token_ids: &[u32], + _skip_special_tokens: bool, + ) -> vllm_tokenizer::Result { + Ok(token_ids + .iter() + .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) + .collect()) + } + + fn token_to_id(&self, token: &str) -> Option { + match token { + MESSAGE_MODEL => Some(200001), + "<|content_text|>" => Some(200004), + "<|content_model_end_sampling|>" => Some(200006), + "<|content_thinking|>" => Some(200008), + "<|end_message|>" => Some(200010), + "<|content_tool_error|>" => Some(200022), + "<|content_invoke_tool_json|>" => Some(200049), + "<|content_invoke_tool_text|>" => Some(200057), + _ => None, + } + } + + fn id_to_token(&self, id: u32) -> Option { + let token = match id { + 200001 => MESSAGE_MODEL, + 200004 => "<|content_text|>", + 200006 => "<|content_model_end_sampling|>", + 200008 => "<|content_thinking|>", + 200010 => "<|end_message|>", + 200022 => "<|content_tool_error|>", + 200049 => "<|content_invoke_tool_json|>", + 200057 => "<|content_invoke_tool_text|>", + _ => return None, + }; + Some(token.to_string()) + } + + fn is_special_id(&self, token_id: u32) -> bool { + (199999..=200057).contains(&token_id) + } + } + + trait UnifiedParserTestExt { + fn parse_chunk(&mut self, chunk: &str) -> super::Result; + fn parse_complete(&mut self, text: &str) -> super::Result; + } + + impl UnifiedParserTestExt for T { + fn parse_chunk(&mut self, chunk: &str) -> super::Result { + let mut output = UnifiedParserOutput::default(); + self.parse_into(chunk, &mut output)?; + Ok(output) + } + + fn parse_complete(&mut self, text: &str) -> super::Result { + let mut output = self.parse_chunk(text)?; + output.append(self.finish()?); + Ok(output) + } + } + + trait UnifiedParserOutputTestExt { + fn normal_text(&self) -> String; + fn reasoning_text(&self) -> String; + fn calls(&self) -> Vec; + } + + impl UnifiedParserOutputTestExt for UnifiedParserOutput { + fn normal_text(&self) -> String { + self.events + .iter() + .filter_map(|event| match event { + UnifiedParserEvent::Text(text) => Some(text.as_str()), + _ => None, + }) + .collect() + } + + fn reasoning_text(&self) -> String { + self.events + .iter() + .filter_map(|event| match event { + UnifiedParserEvent::Reasoning(text) => Some(text.as_str()), + _ => None, + }) + .collect() + } + + fn calls(&self) -> Vec { + self.events + .iter() + .filter_map(|event| match event { + UnifiedParserEvent::ToolCall(call) => Some(call.clone()), + _ => None, + }) + .collect() + } + } + + fn test_tools() -> Vec { + vec![Tool { + name: "get_weather".to_string(), + description: Some("Get weather".to_string()), + parameters: serde_json::json!({ + "type": "object", + "properties": { + "city": {"type": "string"} + }, + }), + strict: None, + }] + } + + fn test_parser() -> InklingUnifiedParser { + InklingUnifiedParser::new(&test_tools(), Arc::new(FakeTokenizer)).unwrap() + } + + fn collect_stream(chunks: &[&str]) -> UnifiedParserOutput { + let mut parser = test_parser(); + let mut output = UnifiedParserOutput::default(); + for chunk in chunks { + output.append(parser.parse_chunk(chunk).unwrap()); + } + output.append(parser.finish().unwrap()); + output + } + + #[test] + fn inkling_streaming_emits_reasoning_then_text() { + let output = collect_stream(&[concat!( + "<|content_thinking|>reason<|end_message|>", + "<|message_model|><|content_text|>answer<|end_message|>", + "<|content_model_end_sampling|>" + )]); + + assert_eq!(output.reasoning_text(), "reason"); + assert_eq!(output.normal_text(), "answer"); + assert!(output.calls().is_empty()); + } + + #[test] + fn inkling_streaming_holds_split_markers() { + let output = collect_stream(&[ + "<|content_thin", + "king|>rea", + "son<|end_mes", + "sage|><|message_", + "model|><|content_text|>answer", + "<|end_message|>", + ]); + + assert_eq!(output.reasoning_text(), "reason"); + assert_eq!(output.normal_text(), "answer"); + } + + #[test] + fn inkling_streaming_accepts_content_blocks_without_message_start() { + let output = collect_stream(&[concat!( + "<|content_thinking|>reason<|end_message|>", + "<|content_text|>answer<|end_message|>", + )]); + + assert_eq!(output.reasoning_text(), "reason"); + assert_eq!(output.normal_text(), "answer"); + } + + #[test] + fn inkling_tool_json_streams_name_then_argument_deltas() { + let mut parser = test_parser(); + let chunks = [ + "<|message_model|><|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":", + "{\"city\":", + "\"SF\"", + "}}<|end_message|>", + ]; + + let mut output = UnifiedParserOutput::default(); + let mut observed_args = Vec::new(); + for chunk in chunks { + let next = parser.parse_chunk(chunk).unwrap(); + observed_args.extend( + next.calls() + .into_iter() + .filter(|call| call.name.is_none()) + .map(|call| call.arguments), + ); + output.append(next); + } + output.append(parser.finish().unwrap()); + + let calls = output.calls(); + assert_eq!(calls[0].name.as_deref(), Some("get_weather")); + assert_eq!(observed_args, ["{\"city\":", "\"SF\"", "}"]); + assert_eq!( + calls.iter().map(|call| call.arguments.as_str()).collect::(), + "{\"city\":\"SF\"}" + ); + assert!(output.normal_text().is_empty()); + } + + #[test] + fn inkling_discards_tool_name_from_message_header() { + let output = collect_stream(&[concat!( + "<|message_model|>get_weather<|content_invoke_tool_json|>", + "{\"name\":\"get_weather\",\"args\":{}}<|end_message|>" + )]); + + assert!(output.normal_text().is_empty()); + assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather")); + } + + #[test] + fn inkling_streaming_handles_multiple_tool_blocks() { + let output = collect_stream(&[concat!( + "<|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":{\"city\":\"SF\"}}<|end_message|>", + "<|message_model|><|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":{\"city\":\"NYC\"}}<|end_message|>" + )]); + + let calls = output.calls(); + assert_eq!(calls.iter().filter(|call| call.name.is_some()).count(), 2); + assert_eq!(calls[0].tool_index, 0); + assert_eq!(calls[2].tool_index, 1); + } + + #[test] + fn inkling_initialize_open_text_prompt_starts_in_text() { + let mut parser = test_parser(); + parser.initialize(&[200004]).unwrap(); + + let output = parser.parse_complete("answer<|end_message|>").unwrap(); + + assert_eq!(output.normal_text(), "answer"); + assert!(output.reasoning_text().is_empty()); + } + + #[test] + fn inkling_initialize_open_reasoning_prompt_starts_in_reasoning() { + let mut parser = test_parser(); + parser.initialize(&[200008]).unwrap(); + + let output = parser.parse_complete("reason<|end_message|>").unwrap(); + + assert_eq!(output.reasoning_text(), "reason"); + assert!(output.normal_text().is_empty()); + } + + #[test] + fn inkling_initialize_model_opener_starts_in_message_header() { + let mut parser = test_parser(); + parser.initialize(&[200001]).unwrap(); + + let output = parser + .parse_complete(concat!( + "get_weather<|content_invoke_tool_json|>", + "{\"name\":\"get_weather\",\"args\":{}}<|end_message|>" + )) + .unwrap(); + + assert!(output.normal_text().is_empty()); + assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather")); + } + + #[test] + fn inkling_plain_text_falls_through_as_text() { + let output = collect_stream(&["plain ", "answer"]); + + assert_eq!(output.normal_text(), "plain answer"); + assert!(output.reasoning_text().is_empty()); + } + + #[test] + fn inkling_raw_tool_text_and_tool_error_are_visible_text() { + let output = collect_stream(&[concat!( + "<|content_invoke_tool_text|>search SF<|end_message|>", + "<|content_tool_error|>failed<|end_message|>" + )]); + + assert_eq!(output.normal_text(), "search SFfailed"); + assert!(output.calls().is_empty()); + } + + #[test] + fn inkling_finish_fails_incomplete_tool_call() { + let mut parser = test_parser(); + parser + .parse_chunk("<|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":{\"city\"") + .unwrap(); + + let error = parser.finish().unwrap_err(); + + assert!(error.as_report().to_string().contains("incomplete Inkling tool call")); + } + + #[test] + fn inkling_rejects_non_object_args() { + let mut parser = test_parser(); + let error = parser + .parse_chunk("<|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":42") + .unwrap_err(); + + assert!(error.as_report().to_string().contains("JSON object argument")); + } + + #[test] + fn inkling_missing_token_fails_create() { + struct MissingTokenizer; + + impl Tokenizer for MissingTokenizer { + fn encode( + &self, + _text: &str, + _add_special_tokens: bool, + ) -> vllm_tokenizer::Result> { + Ok(vec![]) + } + + fn decode( + &self, + _token_ids: &[u32], + _skip_special_tokens: bool, + ) -> vllm_tokenizer::Result { + Ok(String::new()) + } + + fn token_to_id(&self, token: &str) -> Option { + match token { + MESSAGE_MODEL => Some(200001), + CONTENT_TEXT => Some(200004), + _ => None, + } + } + + fn id_to_token(&self, id: u32) -> Option { + match id { + 200001 => Some(MESSAGE_MODEL.to_string()), + 200004 => Some(CONTENT_TEXT.to_string()), + _ => None, + } + } + } + + let error = match InklingUnifiedParser::new(&[], Arc::new(MissingTokenizer)) { + Ok(_) => panic!("expected parser creation to fail"), + Err(error) => error, + }; + + assert!(error.as_report().to_string().contains(CONTENT_THINKING)); + } +} diff --git a/rust/src/parser/src/unified/mod.rs b/rust/src/parser/src/unified/mod.rs index 1fec8ac09b..d5ecb64929 100644 --- a/rust/src/parser/src/unified/mod.rs +++ b/rust/src/parser/src/unified/mod.rs @@ -5,9 +5,11 @@ mod combined; mod gemma4; +mod inkling; pub use combined::CombinedParser; pub use gemma4::Gemma4UnifiedParser; +pub use inkling::InklingUnifiedParser; use thiserror::Error; use thiserror_ext::Macro; use vllm_tokenizer::DynTokenizer; @@ -208,3 +210,10 @@ pub enum UnifiedParserError { #[error(transparent)] Tool(#[from] ToolParserError), } + +/// Returns the ID for the given token, or an error if it's not found. +fn token_id(tokenizer: &dyn vllm_tokenizer::Tokenizer, token: &str) -> Result { + tokenizer.token_to_id(token).ok_or_else(|| UnifiedParserError::MissingToken { + token: token.to_string(), + }) +} diff --git a/rust/src/text/src/backend/hf/config.rs b/rust/src/text/src/backend/hf/config.rs index 140bfdc304..2a69632c87 100644 --- a/rust/src/text/src/backend/hf/config.rs +++ b/rust/src/text/src/backend/hf/config.rs @@ -95,6 +95,7 @@ impl HfSpecialTokens { pub struct ModelConfig { model_type: Option, vocab_size: Option, + eos_token_id: Option, num_experts: Option, moe_num_experts: Option, n_routed_experts: Option, @@ -125,12 +126,16 @@ pub(super) enum OneOrManyTokenIds { } impl OneOrManyTokenIds { - pub(super) fn into_set(self) -> BTreeSet { + pub(super) fn as_slice(&self) -> &[u32] { match self { - Self::One(id) => BTreeSet::from([id]), - Self::Many(ids) => ids.into_iter().collect(), + Self::One(id) => std::slice::from_ref(id), + Self::Many(ids) => ids.as_slice(), } } + + pub(super) fn into_set(self) -> BTreeSet { + self.as_slice().iter().copied().collect() + } } /// Hugging Face configs may expose the expert count either as one integer or @@ -195,6 +200,18 @@ impl ModelConfig { } } + /// Return the effective model-side EOS token ids, following the same + /// simplified text-config selection as `vocab_size`. + pub(super) fn eos_token_ids(&self) -> &[u32] { + if let Some(eos_token_id) = self.eos_token_id.as_ref() { + eos_token_id.as_slice() + } else if let Some(text_config) = self.text_config.as_deref() { + text_config.eos_token_ids() + } else { + &[] + } + } + /// Match Python's current expert-count priority on the selected text /// config. /// @@ -354,6 +371,32 @@ mod tests { assert_eq!(config.vocab_size().unwrap(), 151936); } + #[test] + fn model_config_reads_top_level_eos_token_ids() { + let single: ModelConfig = serde_json::from_str(r#"{"eos_token_id":151645}"#).unwrap(); + let many: ModelConfig = + serde_json::from_str(r#"{"eos_token_id":[128001,128008,128009]}"#).unwrap(); + let null: ModelConfig = serde_json::from_str(r#"{"eos_token_id":null}"#).unwrap(); + + assert_eq!(single.eos_token_ids(), &[151645]); + assert_eq!(many.eos_token_ids(), &[128001, 128008, 128009]); + assert!(null.eos_token_ids().is_empty()); + } + + #[test] + fn model_config_uses_nested_eos_token_ids_when_top_level_is_absent() { + let config: ModelConfig = serde_json::from_str( + r#"{ + "text_config": { + "eos_token_id": [59246, 59253, 59255] + } + }"#, + ) + .unwrap(); + + assert_eq!(config.eos_token_ids(), &[59246, 59253, 59255]); + } + #[test] fn model_config_rejects_missing_vocab_size() { let config: ModelConfig = serde_json::from_str(r#"{}"#).unwrap(); diff --git a/rust/src/text/src/backend/hf/mod.rs b/rust/src/text/src/backend/hf/mod.rs index a145ecc39b..49ae5dbd6b 100644 --- a/rust/src/text/src/backend/hf/mod.rs +++ b/rust/src/text/src/backend/hf/mod.rs @@ -8,7 +8,9 @@ use std::collections::BTreeSet; use std::sync::Arc; use tracing::info; -use vllm_tokenizer::{DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer}; +use vllm_tokenizer::{ + DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer, Tokenizer, +}; use self::config::{GenerationConfig, load_generation_config}; pub use self::config::{ @@ -56,23 +58,15 @@ impl HfTextBackend { pub fn from_resolved_model_files(files: ResolvedModelFiles, model_id: String) -> Result { let tokenizer_config = load_tokenizer_config(files.tokenizer_config_path.as_deref())?; let tokenizer = load_tokenizer(&files.tokenizer)?; - let primary_eos_token_id = tokenizer_config - .special_tokens - .eos_token - .as_ref() - .and_then(|token| tokenizer.token_to_id(token.as_str())); - let model_config = load_model_config(files.config_path.as_deref())?; let model_vocab_size = model_config.vocab_size()? as usize; let generation_config = load_generation_config(files.generation_config_path.as_deref())?; - let mut extra_eos_token_ids = generation_config - .eos_token_id - .clone() - .map(|value| value.into_set()) - .unwrap_or_default(); - if let Some(primary_eos_token_id) = primary_eos_token_id { - extra_eos_token_ids.remove(&primary_eos_token_id); - } + let (primary_eos_token_id, extra_eos_token_ids) = resolve_eos_token_ids( + &tokenizer_config, + &model_config, + &generation_config, + tokenizer.as_ref(), + ); info!( model_id, @@ -98,6 +92,42 @@ impl HfTextBackend { } } +/// Resolve EOS hints from tokenizer, model, and generation configs. +/// +/// Resolution rules: +/// 1. Use the tokenizer-side EOS token as the primary EOS when it resolves to a +/// token id. +/// 2. Fall back to the first model-config EOS id when the tokenizer config does +/// not provide a primary EOS. +/// 3. Keep any remaining model/generation EOS ids as extra stop-token ids so +/// they still participate in stopping and min-token handling. +fn resolve_eos_token_ids( + tokenizer_config: &HfTokenizerConfig, + model_config: &ModelConfig, + generation_config: &GenerationConfig, + tokenizer: &dyn Tokenizer, +) -> (Option, BTreeSet) { + let model_config_eos_token_ids = model_config.eos_token_ids(); + let primary_eos_token_id = tokenizer_config + .special_tokens + .eos_token + .as_ref() + .and_then(|token| tokenizer.token_to_id(token.as_str())) + .or_else(|| model_config_eos_token_ids.first().copied()); + + let mut extra_eos_token_ids = generation_config + .eos_token_id + .clone() + .map(|value| value.into_set()) + .unwrap_or_default(); + extra_eos_token_ids.extend(model_config_eos_token_ids.iter().copied()); + if let Some(primary_eos_token_id) = primary_eos_token_id { + extra_eos_token_ids.remove(&primary_eos_token_id); + } + + (primary_eos_token_id, extra_eos_token_ids) +} + impl TextBackend for HfTextBackend { fn tokenizer(&self) -> DynTokenizer { self.tokenizer.clone() @@ -128,3 +158,77 @@ impl TextBackend for HfTextBackend { }) } } + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use super::{GenerationConfig, HfTokenizerConfig, ModelConfig, resolve_eos_token_ids}; + use vllm_tokenizer::Tokenizer; + + struct FakeTokenizer; + + impl Tokenizer for FakeTokenizer { + fn encode( + &self, + _text: &str, + _add_special_tokens: bool, + ) -> vllm_tokenizer::Result> { + Ok(vec![]) + } + + fn decode( + &self, + _token_ids: &[u32], + _skip_special_tokens: bool, + ) -> vllm_tokenizer::Result { + Ok(String::new()) + } + + fn token_to_id(&self, token: &str) -> Option { + (token == "").then_some(2) + } + + fn id_to_token(&self, id: u32) -> Option { + (id == 2).then(|| "".to_string()) + } + } + + #[test] + fn eos_resolution_uses_model_config_when_tokenizer_has_no_eos() { + let tokenizer_config: HfTokenizerConfig = serde_json::from_str("{}").unwrap(); + let model_config: ModelConfig = + serde_json::from_str(r#"{"eos_token_id":[200006,200010]}"#).unwrap(); + let generation_config: GenerationConfig = serde_json::from_str("{}").unwrap(); + + let (primary, extra) = resolve_eos_token_ids( + &tokenizer_config, + &model_config, + &generation_config, + &FakeTokenizer, + ); + + assert_eq!(primary, Some(200006)); + assert_eq!(extra, BTreeSet::from([200010])); + } + + #[test] + fn eos_resolution_keeps_tokenizer_eos_primary() { + let tokenizer_config: HfTokenizerConfig = + serde_json::from_str(r#"{"eos_token":""}"#).unwrap(); + let model_config: ModelConfig = + serde_json::from_str(r#"{"eos_token_id":[2,200006]}"#).unwrap(); + let generation_config: GenerationConfig = + serde_json::from_str(r#"{"eos_token_id":[2,200010]}"#).unwrap(); + + let (primary, extra) = resolve_eos_token_ids( + &tokenizer_config, + &model_config, + &generation_config, + &FakeTokenizer, + ); + + assert_eq!(primary, Some(2)); + assert_eq!(extra, BTreeSet::from([200006, 200010])); + } +} diff --git a/setup.py b/setup.py index 6419e3073e..7bc4d42c92 100644 --- a/setup.py +++ b/setup.py @@ -445,6 +445,17 @@ class cmake_build_ext(build_ext): dirs_exist_ok=True, ) + tml_fa4_build = os.path.join( + self.build_lib, "vllm", "third_party", "tml_fa4" + ) + if os.path.exists(tml_fa4_build): + print(f"Copying {tml_fa4_build} to vllm/third_party/tml_fa4") + shutil.copytree( + tml_fa4_build, + "vllm/third_party/tml_fa4", + dirs_exist_ok=True, + ) + class precompiled_build_ext(build_ext): """Disables extension building when using precompiled binaries.""" @@ -803,6 +814,7 @@ class precompiled_wheel_utils: # DeepGEMM: extract all files (.py, .so, .cuh, .h, .hpp, etc.) deep_gemm_regex = re.compile(r"vllm/third_party/deep_gemm/.*") fmha_sm100_regex = re.compile(r"vllm/third_party/fmha_sm100/.*") + tml_fa4_regex = re.compile(r"vllm/third_party/tml_fa4/.*") file_members = [] for member in wheel.filelist: if member.filename in exact_members: @@ -828,6 +840,7 @@ class precompiled_wheel_utils: or triton_kernels_regex.match(member.filename) or flashmla_regex.match(member.filename) or deep_gemm_regex.match(member.filename) + or tml_fa4_regex.match(member.filename) or fmha_sm100_regex.match(member.filename) ): file_members.append(member) @@ -1146,6 +1159,8 @@ if _is_cuda(): ext_modules.append(CMakeExtension(name="vllm._qutlass_C", optional=True)) # fmha_sm100 is a Python/CuTe-DSL package installed into vllm.third_party. ext_modules.append(CMakeExtension(name="vllm.fmha_sm100", optional=True)) + # tml-fa4 is copied into an isolated vllm.third_party package. + ext_modules.append(CMakeExtension(name="vllm.tml_fa4", optional=True)) if _is_cpu(): import platform diff --git a/tests/kernels/test_ll_bf16_gemm.py b/tests/kernels/test_ll_bf16_gemm.py index f530f6f741..e71c89aa0d 100644 --- a/tests/kernels/test_ll_bf16_gemm.py +++ b/tests/kernels/test_ll_bf16_gemm.py @@ -190,8 +190,8 @@ def test_arbitrary_N_splitk(N): @pytest.mark.parametrize( "N,K", - [(256, 7168), (256, 14400), (8, 4096), (384, 7168)], - ids=["DSV3", "DSV4-Flash", "Mixtral", "DSV4-Pro"], + [(256, 7168), (256, 14400), (8, 4096), (384, 7168), (264, 6144)], + ids=["DSV3", "DSV4-Flash", "Mixtral", "DSV4-Pro", "Inkling"], ) def test_single_token(N, K): torch.manual_seed(42) @@ -202,6 +202,15 @@ def test_single_token(N, K): _assert_close(out, _ref(a, b), context=f"M=1 {N}x{K}") +def test_inkling_max_tokens(): + torch.manual_seed(42) + a = torch.randn(64, 6144, dtype=torch.bfloat16, device="cuda") + b = torch.randn(264, 6144, dtype=torch.bfloat16, device="cuda") + out = _gemm(a, b) + assert out.shape == (64, 264) + _assert_close(out, _ref(a, b), context="Inkling M=64") + + # ================================================================= # Numerical robustness # ================================================================= diff --git a/tests/models/inkling/test_contract_validation.py b/tests/models/inkling/test_contract_validation.py new file mode 100644 index 0000000000..535359996d --- /dev/null +++ b/tests/models/inkling/test_contract_validation.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import numpy as np +import pytest + +from vllm.config.compilation import CompilationConfig, CUDAGraphMode +from vllm.models.inkling.common.mm_preprocess import InklingMultiModalDataParser +from vllm.models.inkling.configs import ( + InklingAudioConfig, + InklingVisionConfig, +) +from vllm.models.inkling.nvidia.sconv_swa_attn import ( + InklingSconvMetadataBuilder, +) +from vllm.v1.attention.backend import AttentionCGSupport + + +@pytest.mark.parametrize( + ("config_cls", "kwargs", "missing"), + [ + (InklingAudioConfig, {"decoder_dmodel": 16}, "n_mel_bins"), + (InklingVisionConfig, {"decoder_dmodel": 16}, "vision_encoder_type"), + ], +) +def test_enabled_tower_requires_architecture_fields(config_cls, kwargs, missing): + with pytest.raises(ValueError, match=missing): + config_cls(**kwargs) + + +def test_inkling_raw_2d_audio_is_rejected_as_ambiguous(): + parser = InklingMultiModalDataParser(target_sr=16_000, target_channels=1) + with pytest.raises(ValueError, match="ambiguous channel layout"): + parser._parse_audio_data(np.zeros((2, 100), dtype=np.float32)) + + +def test_inkling_supports_full_decode_only_cudagraphs(): + support = InklingSconvMetadataBuilder.get_cudagraph_support + assert support(None, None) == AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE + + compilation_config = CompilationConfig( + cudagraph_mode=CUDAGraphMode.FULL, + splitting_ops=[], + ) + resolved_mode = compilation_config.resolve_cudagraph_mode_and_sizes( + AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE, + "InklingSconvBackend", + ) + + assert resolved_mode == CUDAGraphMode.FULL_DECODE_ONLY diff --git a/tests/models/inkling/test_fa4_rel_attention.py b/tests/models/inkling/test_fa4_rel_attention.py new file mode 100644 index 0000000000..86cacd9da8 --- /dev/null +++ b/tests/models/inkling/test_fa4_rel_attention.py @@ -0,0 +1,381 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Correctness test for the Inkling FA4 relative-attention score-mod kernel. + +Checks ``inkling_fa4_rel_attention`` against a pure-PyTorch reference that +implements the relative bias exactly as documented in the Inkling architecture +guide:: + + logit(i, j, h) = (1 / head_dim) * dot(q[i, h], k[j, h]) + rel_bias(i, j, h) + rel_bias(i, j, h) = rel_logits[i, h, i - j] if 0 <= i - j < rel_extent + = 0 otherwise + +with causal (and optionally sliding-window) masking handled by the backend. +""" + +import pytest +import torch + +from vllm.models.inkling.nvidia.attention import ( + InklingAttention, + compute_log_scaling_tau, +) +from vllm.models.inkling.nvidia.ops.fa4_rel_attention import ( + inkling_fa4_num_splits, + inkling_fa4_rel_attention, +) +from vllm.platforms import current_platform +from vllm.platforms.interface import DeviceCapability + +_cap = current_platform.get_device_capability() if current_platform.is_cuda() else None + +NUM_HEADS = [(4, 4), (8, 2)] # (num_heads, num_kv_heads) +GLOBAL_REL_EXTENTS = [128, 1024] +LOCAL_REL_EXTENTS = [128, 256] +HEAD_DIM = 128 +BLOCK_SIZE = 16 +DTYPE = torch.bfloat16 + + +def test_log_scaling_tau_matches_reference(): + positions = torch.tensor([0, 127999, 128000, 999999], dtype=torch.int64) + actual = compute_log_scaling_tau(positions, 128000, 0.1) + expected = 1.0 + 0.1 * torch.log( + torch.clamp((positions + 1).float() / 128000.0, min=1.0) + ) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +def test_split_packed_kv_cache(): + attention = InklingAttention.__new__(InklingAttention) + torch.nn.Module.__init__(attention) + attention.head_dim = 8 + attention.kv_cache = torch.arange(2 * 3 * 4 * 16).reshape(2, 3, 4, 16) + + key_cache, value_cache = attention._split_kv_cache() + + assert key_cache.shape == value_cache.shape == (2, 4, 3, 8) + torch.testing.assert_close(key_cache, attention.kv_cache[..., :8].transpose(1, 2)) + torch.testing.assert_close(value_cache, attention.kv_cache[..., 8:].transpose(1, 2)) + + +def test_num_splits_hopper_is_unsplit(monkeypatch): + monkeypatch.setattr( + current_platform, + "get_device_capability", + lambda: DeviceCapability(major=9, minor=0), + ) + assert ( + inkling_fa4_num_splits( + is_local=False, + batch_size=1, + max_query_len=1, + num_heads=16, + num_kv_heads=2, + max_kv_len=1_048_576, + ) + == 1 + ) + + +@pytest.fixture +def blackwell_platform(monkeypatch): + monkeypatch.setattr( + current_platform, + "get_device_capability", + lambda: DeviceCapability(major=10, minor=0), + ) + + +@pytest.mark.parametrize( + ("batch_size", "max_query_len", "expected"), + [ + (1, 1, (16, 32, 128, 128)), + (8, 1, (2, 4, 8, 16)), + (32, 1, (1, 1, 2, 4)), + (1, 128, (2, 4, 8, 16)), + (1, 2048, (1, 1, 1, 1)), + ], +) +def test_num_splits_all_tp(blackwell_platform, batch_size, max_query_len, expected): + actual = tuple( + inkling_fa4_num_splits( + is_local=False, + batch_size=batch_size, + max_query_len=max_query_len, + num_heads=64 // tp, + num_kv_heads=8 // tp, + max_kv_len=131072, + ) + for tp in (1, 2, 4, 8) + ) + assert actual == expected + + +@pytest.mark.parametrize("tp", [1, 2, 4, 8]) +def test_num_splits_local_is_unsplit(tp): + assert ( + inkling_fa4_num_splits( + is_local=True, + batch_size=1, + max_query_len=1, + num_heads=64 // tp, + num_kv_heads=16 // tp, + max_kv_len=512, + ) + == 1 + ) + + +@pytest.mark.parametrize( + ("max_kv_len", "expected"), + [(8192, 32), (65536, 64), (1048576, 128)], +) +@pytest.mark.parametrize("tp", [4, 8]) +def test_num_splits_long_context_bound(blackwell_platform, tp, max_kv_len, expected): + assert ( + inkling_fa4_num_splits( + is_local=False, + batch_size=1, + max_query_len=1, + num_heads=64 // tp, + num_kv_heads=8 // tp, + max_kv_len=max_kv_len, + ) + == expected + ) + + +def _ref_rel_attn( + q: torch.Tensor, # [total_q, H, D] + key_cache: torch.Tensor, # [num_blocks, block, Hkv, D] + value_cache: torch.Tensor, + rel_logits: torch.Tensor, # [total_q, H, rel_extent] + *, + q_lens: list[int], + kv_lens: list[int], + block_table: torch.Tensor, + scale: float, + rel_extent: int, + window_left: int | None, +) -> torch.Tensor: + num_kv_heads = key_cache.shape[2] + num_heads = q.shape[1] + g = num_heads // num_kv_heads + bt = block_table.cpu().numpy() + out = torch.empty_like(q) + + start = 0 + for i, (ql, kl) in enumerate(zip(q_lens, kv_lens)): + qi = q[start : start + ql].float() # [ql, H, D] + rl = rel_logits[start : start + ql].float() # [ql, H, rel_extent] + + nblk = (kl + BLOCK_SIZE - 1) // BLOCK_SIZE + blk = bt[i, :nblk] + k = key_cache[blk].reshape(-1, num_kv_heads, HEAD_DIM)[:kl].float() + v = value_cache[blk].reshape(-1, num_kv_heads, HEAD_DIM)[:kl].float() + k = k.repeat_interleave(g, dim=1) # [kl, H, D] + v = v.repeat_interleave(g, dim=1) + + # [H, ql, kl] + scores = torch.einsum("qhd,khd->hqk", qi, k) * scale + + dev = q.device + qpos = torch.arange(ql, device=dev).view(ql, 1) + (kl - ql) # query pos + kpos = torch.arange(kl, device=dev).view(1, kl) + dist = qpos - kpos # [ql, kl] = i - j + + # Relative bias: rel_logits[i, h, dist] when 0 <= dist < rel_extent. + in_rng = (dist >= 0) & (dist < rel_extent) # [ql, kl] + idx = dist.clamp(0, rel_extent - 1) + # gather per head: bias[h, i, j] = rl[i, h, idx[i, j]] + bias = rl.permute(1, 0, 2).gather( # [H, ql, rel_extent] + 2, idx.unsqueeze(0).expand(num_heads, -1, -1) + ) # [H, ql, kl] + bias = torch.where(in_rng.unsqueeze(0), bias, torch.zeros_like(bias)) + scores = scores + bias + + mask = dist < 0 # causal + if window_left is not None: + mask = mask | (dist > window_left) + scores.masked_fill_(mask.unsqueeze(0), float("-inf")) + + probs = torch.softmax(scores, dim=-1) + out[start : start + ql] = torch.einsum("hqk,khd->qhd", probs, v).to(q.dtype) + start += ql + return out + + +def _run_case(seq_lens, num_heads, num_kv_heads, rel_extent, window_left, seed=0): + torch.manual_seed(seed) + device = "cuda" + q_lens = [s[0] for s in seq_lens] + kv_lens = [s[1] for s in seq_lens] + total_q = sum(q_lens) + num_seqs = len(seq_lens) + + scale = 1.0 / HEAD_DIM + + # q/k are RMS-normed in the model (unit-ish norm); normalize here so the + # logit magnitudes are realistic and the bias is not numerically dwarfed. + q = torch.randn(total_q, num_heads, HEAD_DIM, device=device, dtype=DTYPE) + q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE) + + # Paged KV cache. + max_blocks = (max(kv_lens) + BLOCK_SIZE - 1) // BLOCK_SIZE + num_blocks = num_seqs * max_blocks + 1 + key_cache = torch.randn( + num_blocks, BLOCK_SIZE, num_kv_heads, HEAD_DIM, device=device, dtype=DTYPE + ) + key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE) + value_cache = torch.randn( + num_blocks, BLOCK_SIZE, num_kv_heads, HEAD_DIM, device=device, dtype=DTYPE + ) + + # Distinct blocks per sequence (block 0 left as a never-referenced pad). + block_table = torch.zeros(num_seqs, max_blocks, dtype=torch.int32, device=device) + for i in range(num_seqs): + block_table[i] = torch.arange( + 1 + i * max_blocks, 1 + (i + 1) * max_blocks, dtype=torch.int32 + ) + + cu_seqlens_q = torch.tensor( + [0, *torch.cumsum(torch.tensor(q_lens), 0).tolist()], + dtype=torch.int32, + device=device, + ) + cache_seqlens = torch.tensor(kv_lens, dtype=torch.int32, device=device) + + rel_logits = torch.randn(total_q, num_heads, rel_extent, device=device, dtype=DTYPE) + + window_size = (-1, -1) if window_left is None else (window_left, 0) + + preallocated_out = torch.empty_like(q) + out = inkling_fa4_rel_attention( + q, + key_cache, + value_cache, + block_table=block_table, + cache_seqlens=cache_seqlens, + cu_seqlens_q=cu_seqlens_q, + max_seqlen_q=max(q_lens), + softmax_scale=scale, + causal=True, + window_size=window_size, + rel_extent=rel_extent, + rel_logits=rel_logits, + out=preallocated_out, + ) + assert out.data_ptr() == preallocated_out.data_ptr() + out = out.view(total_q, num_heads, HEAD_DIM) + + ref = _ref_rel_attn( + q, + key_cache, + value_cache, + rel_logits, + q_lens=q_lens, + kv_lens=kv_lens, + block_table=block_table, + scale=scale, + rel_extent=rel_extent, + window_left=window_left, + ) + + torch.testing.assert_close(out.float(), ref.float(), atol=2e-2, rtol=2e-2) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA") +@pytest.mark.skipif( + _cap is None or _cap.major < 9, + reason="FA4 score-mod requires Hopper+ (SM90+)", +) +@pytest.mark.parametrize("num_heads", NUM_HEADS) +@pytest.mark.parametrize( + "seq_lens", + [ + [(64, 64)], # single full prefill + [(64, 64), (33, 33), (17, 17)], # ragged prefill batch + [(512, 512)], # seq_len >> rel_extent (most keys get zero bias) + [(300, 300), (512, 512), (129, 129)], # large ragged batch + ], +) +@pytest.mark.parametrize("rel_extent", GLOBAL_REL_EXTENTS) +@torch.inference_mode() +def test_full_attention(seq_lens, num_heads, rel_extent): + # rel_extent=128 exercises the out-of-range (zero bias) path; 1024 covers all. + # With the 512-token cases and rel_extent=128, query/seq lengths are far + # larger than rel_extent so the vast majority of (i, j) pairs are out of + # range and must contribute zero bias. + _run_case(seq_lens, num_heads[0], num_heads[1], rel_extent, window_left=None) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA") +@pytest.mark.skipif( + _cap is None or _cap.major < 9, + reason="FA4 score-mod requires Hopper+ (SM90+)", +) +@pytest.mark.parametrize("num_heads", NUM_HEADS) +@pytest.mark.parametrize( + "seq_lens", + [ + [(200, 512)], # chunked prefill: q_len=200 (> rel_extent), 312 cached + [(200, 512), (50, 300), (1, 400)], # mixed chunked + decode + ], +) +@pytest.mark.parametrize("rel_extent", GLOBAL_REL_EXTENTS) +@torch.inference_mode() +def test_chunked_prefill(seq_lens, num_heads, rel_extent): + # q_len < kv_len with q_len itself larger than rel_extent (for the 128 case): + # exercises the seqlen_k - seqlen_q offset together with the out-of-range path. + _run_case(seq_lens, num_heads[0], num_heads[1], rel_extent, window_left=None) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA") +@pytest.mark.skipif( + _cap is None or _cap.major < 9, + reason="FA4 score-mod requires Hopper+ (SM90+)", +) +@pytest.mark.parametrize("num_heads", NUM_HEADS) +@pytest.mark.parametrize( + "seq_lens", + [ + [(64, 64), (40, 40)], # seq_len > window + [(512, 512), (300, 300)], # seq_len/query_len >> window + [(1, 512)], # decode with kv_len >> window + ], +) +@pytest.mark.parametrize("local_extent", LOCAL_REL_EXTENTS) +@torch.inference_mode() +def test_sliding_window(seq_lens, num_heads, local_extent): + # Local layers use window_size=(local_extent-1, 0) and rel_extent==local_extent. + # With the 512-token cases, query/seq lengths far exceed the window so most + # keys are masked out by the sliding window. + _run_case( + seq_lens, + num_heads[0], + num_heads[1], + rel_extent=local_extent, + window_left=local_extent - 1, + ) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA") +@pytest.mark.skipif( + _cap is None or _cap.major < 9, + reason="FA4 score-mod requires Hopper+ (SM90+)", +) +@pytest.mark.parametrize("num_heads", NUM_HEADS) +@pytest.mark.parametrize( + "seq_lens", + [ + [(1, 50)], + [(1, 50), (1, 7), (1, 200)], + [(1, 512), (1, 333)], # kv_len >> rel_extent + ], +) +@pytest.mark.parametrize("rel_extent", GLOBAL_REL_EXTENTS) +@torch.inference_mode() +def test_decode(seq_lens, num_heads, rel_extent): + # q_len=1 with kv_len>q_len: the score-mod's seqlen_k - seqlen_q offset path. + _run_case(seq_lens, num_heads[0], num_heads[1], rel_extent, window_left=None) diff --git a/tests/models/inkling/test_fa4_warmup.py b/tests/models/inkling/test_fa4_warmup.py new file mode 100644 index 0000000000..eea9798aa9 --- /dev/null +++ b/tests/models/inkling/test_fa4_warmup.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.models.inkling.nvidia.ops.fa4_rel_attention import ( + bucket_max_seqlen_q, + inkling_fa4_num_splits, +) +from vllm.models.inkling.nvidia.ops.fa4_warmup import ( + InklingFA4WarmupConfig, + _iter_compile_units, + _num_warps_bucket, +) + + +def test_bucket_max_seqlen_q(): + assert [bucket_max_seqlen_q(n) for n in range(1, 10)] == [ + 1, + 2, + 4, + 4, + 8, + 8, + 8, + 8, + 16, + ] + + +def test_warmup_enumerates_every_runtime_compile_class(): + config = InklingFA4WarmupConfig( + num_heads=16, + num_kv_heads=2, + head_dim=128, + rel_extent=1024, + window_size=(-1, -1), + is_local=False, + max_kv_len=65536, + dtype=torch.bfloat16, + kv_dtype=torch.bfloat16, + block_size=16, + max_num_reqs=64, + max_num_batched_tokens=192, + ) + warmed_keys = {unit.key[-1] for unit in _iter_compile_units(config)} + + for query_len in range(1, config.max_num_batched_tokens + 1): + max_seqlen_q = bucket_max_seqlen_q(query_len) + max_num_reqs = min( + config.max_num_reqs, + config.max_num_batched_tokens - query_len + 1, + ) + for num_reqs in range(1, max_num_reqs + 1): + num_splits = inkling_fa4_num_splits( + is_local=config.is_local, + batch_size=num_reqs, + max_query_len=max_seqlen_q, + num_heads=config.num_heads, + num_kv_heads=config.num_kv_heads, + max_kv_len=config.max_kv_len, + ) + runtime_key = ( + max_seqlen_q, + num_splits, + _num_warps_bucket(num_reqs) if num_splits > 1 else None, + num_reqs > 1024, + ) + assert runtime_key in warmed_keys diff --git a/tests/models/inkling/test_mm_towers.py b/tests/models/inkling/test_mm_towers.py new file mode 100644 index 0000000000..d969c8101a --- /dev/null +++ b/tests/models/inkling/test_mm_towers.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Correctness tests for the fused Inkling vision/audio tower kernels. + +Each kernel is checked against a pure-PyTorch reference implementing the +towers' original op sequence (native ``rms_norm`` semantics: fp32 variance, +bf16-rounded weight multiply; exact-erf GELU; fp32-accumulated embedding sum). +The fused kernels keep the same accumulation dtype and rounding points, so +outputs differ from the reference only by reduction order — a few bf16 ulps. +""" + +import pytest +import torch +import torch.nn.functional as F + +from vllm.platforms import current_platform + +if not current_platform.is_cuda(): + pytest.skip("requires CUDA", allow_module_level=True) + +from vllm.models.inkling.common.towers import fold_timespace_to_depth +from vllm.models.inkling.nvidia.ops.mm_towers import dmel_embed_sum_norm, rmsnorm_gelu + +DTYPE = torch.bfloat16 + + +def _bf16_spacing(x: torch.Tensor) -> torch.Tensor: + return torch.exp2(torch.floor(torch.log2(x.float().abs().clamp(min=1e-30)))) * ( + 2**-7 + ) + + +def _assert_close_ulps( + got: torch.Tensor, + ref: torch.Tensor, + max_ulps: float = 4.0, + atol: float = 1e-3, + pre_act: torch.Tensor | None = None, +) -> None: + """Fused vs reference differ only by fp32 reduction order, which shows up + as a few bf16 ulps at any magnitude — a fixed rtol misrepresents that, so + compare against the reference's local ulp spacing. ``pre_act`` (the + reference pre-activation) adds derivative-propagated slack: near + ``gelu(x) ~ 0`` an ulp-level input flip legitimately moves the output by + many of ITS (tiny) ulps, bounded by |gelu'| <= 1.13 times the input ulps.""" + g, r = got.float(), ref.float() + tol = torch.clamp(max_ulps * _bf16_spacing(ref), min=atol) + if pre_act is not None: + tol = tol + 2.5 * _bf16_spacing(pre_act) + bad = (g - r).abs() > tol + assert not bad.any(), ( + f"{int(bad.sum())}/{ref.numel()} elements beyond tolerance; " + f"max abs diff {(g - r).abs().max().item():.3e}" + ) + + +def _ref_rms_norm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: + # Matches ir.ops.rms_norm: fp32 normalize, bf16 weight multiply. + x32 = x.float() + var = x32.pow(2).mean(dim=-1, keepdim=True) + xn = x32 * torch.rsqrt(var + eps) + return (xn.to(weight.dtype) * weight).to(x.dtype) + + +@pytest.mark.parametrize("rows", [1, 7, 1200, 38400]) +@pytest.mark.parametrize("dim", [128, 320, 4800, 6144]) +@pytest.mark.parametrize("gelu", [True, False]) +def test_rmsnorm_gelu(rows: int, dim: int, gelu: bool) -> None: + torch.manual_seed(rows * dim) + x = torch.randn(rows, dim, device="cuda", dtype=DTYPE) + w = torch.randn(dim, device="cuda", dtype=DTYPE) + + h = _ref_rms_norm(x, w, 1e-5) + ref = F.gelu(h) if gelu else h + got = rmsnorm_gelu(x, w, 1e-5, gelu=gelu) + + _assert_close_ulps(got, ref, pre_act=h if gelu else None) + + +@pytest.mark.parametrize("n", [1, 5, 64]) +@pytest.mark.parametrize( + "shape,fold", + [ + ((2, 8, 8, 128), (1, 2)), # the real L0 -> L1 vision transition + ((2, 4, 4, 320), (1, 2)), + ((2, 8, 8, 128), (2, 2)), # temporal + spatial fold + ], +) +def test_rmsnorm_gelu_folded_store( + n: int, shape: tuple[int, ...], fold: tuple[int, int] +) -> None: + torch.manual_seed(n) + x = torch.randn(n, *shape, device="cuda", dtype=DTYPE) + w = torch.randn(shape[-1], device="cuda", dtype=DTYPE) + + plain = rmsnorm_gelu(x, w, 1e-5, gelu=True) + ref = fold_timespace_to_depth(plain, *fold) + got = rmsnorm_gelu(x, w, 1e-5, gelu=True, fold=fold) + + # The folded store is a pure permutation of the unfolded output. + assert got.shape == ref.shape + torch.testing.assert_close(got, ref, rtol=0, atol=0) + + +@pytest.mark.parametrize("num_frames", [1, 3, 100, 4097]) +@pytest.mark.parametrize("with_norm", [True, False]) +def test_dmel_embed_sum_norm(num_frames: int, with_norm: bool) -> None: + torch.manual_seed(num_frames) + n_bins, vocab, dim = 80, 16, 6144 + idx = torch.randint( + 0, vocab, (num_frames, n_bins), device="cuda", dtype=torch.int32 + ) + table = torch.randn(n_bins * vocab, dim, device="cuda", dtype=DTYPE) + norm_w = torch.randn(dim, device="cuda", dtype=DTYPE) + + flat = (torch.arange(n_bins, device="cuda", dtype=torch.int32) * vocab).unsqueeze( + 0 + ) + idx + ref = ( + F.embedding(flat.reshape(-1).long(), table) + .reshape(num_frames, n_bins, dim) + .sum(dim=1) + ) + if with_norm: + ref = _ref_rms_norm(ref, norm_w, 1e-6) + + got = dmel_embed_sum_norm(idx, table, norm_w if with_norm else None, 1e-6) + _assert_close_ulps(got, ref) + + +def test_empty_inputs() -> None: + x = torch.empty(0, 320, device="cuda", dtype=DTYPE) + w = torch.randn(320, device="cuda", dtype=DTYPE) + assert rmsnorm_gelu(x, w, 1e-5).shape == (0, 320) + + idx = torch.empty(0, 80, device="cuda", dtype=torch.int32) + table = torch.randn(1280, 6144, device="cuda", dtype=DTYPE) + assert dmel_embed_sum_norm(idx, table, None, 0.0).shape == (0, 6144) diff --git a/tests/models/inkling/test_moe_weight_layout.py b/tests/models/inkling/test_moe_weight_layout.py new file mode 100644 index 0000000000..e46db9be32 --- /dev/null +++ b/tests/models/inkling/test_moe_weight_layout.py @@ -0,0 +1,118 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import torch + +from vllm.models.inkling.nvidia import moe +from vllm.platforms import current_platform + + +def test_gate_loads_directly_into_padded_runtime_weight() -> None: + gate = moe.InklingGate( + d_model=4, + n_routed_experts=5, + n_shared_experts=2, + experts_per_token=2, + route_scale=1.0, + ) + loaded = torch.arange(28, dtype=gate.weight.dtype).reshape(7, 4) + + gate.weight.weight_loader(gate.weight, loaded) + + assert gate.weight.shape == (8, 4) + torch.testing.assert_close(gate.weight[:7], loaded) + torch.testing.assert_close(gate.weight[7], torch.zeros(4)) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA") +@pytest.mark.parametrize(("num_tokens", "expected_calls"), [(1, 1), (64, 1), (65, 0)]) +def test_gate_uses_ll_bf16_gemm_through_token_limit( + monkeypatch, num_tokens, expected_calls +) -> None: + gate = moe.InklingGate( + d_model=8, + n_routed_experts=5, + n_shared_experts=2, + experts_per_token=2, + route_scale=1.0, + ).to(device="cuda", dtype=torch.bfloat16) + hidden_states = torch.randn(num_tokens, 8, device="cuda", dtype=torch.bfloat16) + calls = [] + + def fake_ll_bf16_gemm(x, weight): + calls.append((x, weight)) + return torch.ones(num_tokens, 8, device="cuda", dtype=torch.float32) + + monkeypatch.setattr( + moe.current_platform, "has_device_capability", lambda capability: True + ) + monkeypatch.setattr(moe.ll_bf16, "is_available", lambda: True) + monkeypatch.setattr(moe.ll_bf16, "ll_bf16_gemm", fake_ll_bf16_gemm) + + logits = gate.compute_logits(hidden_states) + + assert len(calls) == expected_calls + if calls: + assert calls[0][0] is hidden_states + assert calls[0][1] is gate.weight + assert logits.shape == (num_tokens, 8) + assert logits.dtype == torch.float32 + + +@pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)]) +def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None: + experts = SimpleNamespace( + w13_input_scale=torch.nn.Parameter(torch.empty(3, 2)), + w2_input_scale=torch.nn.Parameter(torch.empty(3)), + ) + layer = SimpleNamespace(experts=SimpleNamespace(routed_experts=experts)) + + loaded = moe.InklingMoE.load_expert_weight( + layer, + f"experts.{projection}_weight.input_amax", + torch.tensor([amax]), + ) + + scale = getattr(experts, f"{projection}_input_scale") + expected = torch.full_like(scale, amax / (448.0 * 6.0)) + torch.testing.assert_close(scale, expected) + assert loaded == [f"experts.routed_experts.{projection}_input_scale"] + + +def test_sink_down_projection_is_packed_during_load(monkeypatch) -> None: + monkeypatch.setattr(moe, "get_tensor_model_parallel_world_size", lambda: 2) + monkeypatch.setattr(moe, "get_tensor_model_parallel_rank", lambda: 1) + sink = moe.InklingSinkExperts(n_experts=2, d_model=3, d_mlp=8) + loaded = torch.arange(48, dtype=sink.w2_weight.dtype).reshape(2, 3, 8) + + sink.load_weight("w2_weight", loaded) + + expected = loaded[:, :, 4:].permute(1, 0, 2).reshape(3, 8) + torch.testing.assert_close(sink.w2_weight, expected) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA") +def test_sink_packed_weight_forward_matches_expert_sum(monkeypatch) -> None: + monkeypatch.setattr(moe, "get_tensor_model_parallel_world_size", lambda: 2) + monkeypatch.setattr(moe, "get_tensor_model_parallel_rank", lambda: 1) + sink = moe.InklingSinkExperts(n_experts=2, d_model=3, d_mlp=8).to( + device="cuda", dtype=torch.bfloat16 + ) + torch.manual_seed(1) + w13 = torch.randn(2, 16, 3, dtype=torch.bfloat16) + w2 = torch.randn(2, 3, 8, dtype=torch.bfloat16) + sink.load_weight("w13_weight", w13) + sink.load_weight("w2_weight", w2) + x = torch.randn(5, 3, device="cuda", dtype=torch.bfloat16) + gammas = torch.randn(5, 2, device="cuda") + + output = sink(x, gammas) + + raw = torch.einsum("td,efd->tef", x, w13[:, 8:].to("cuda")) + hidden = torch.nn.functional.silu(raw[:, :, 0::2].float()) + hidden = (hidden * raw[:, :, 1::2] * gammas[:, :, None]).to(torch.bfloat16) + expected = torch.einsum("tef,edf->td", hidden, w2[:, :, 4:].to("cuda")) + torch.testing.assert_close(output, expected, rtol=0, atol=0) diff --git a/tests/models/inkling/test_qkvr_prep.py b/tests/models/inkling/test_qkvr_prep.py new file mode 100644 index 0000000000..1e0bf49bb6 --- /dev/null +++ b/tests/models/inkling/test_qkvr_prep.py @@ -0,0 +1,258 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +import vllm.models.inkling.nvidia.ops.sconv as sconv +from vllm.models.inkling.nvidia.ops import qkvr_prep +from vllm.platforms import current_platform + +_cap = current_platform.get_device_capability() if current_platform.is_cuda() else None + + +def _make_inputs(*, is_local: bool, tokens: int = 33, tp_size: int = 4): + torch.manual_seed(0) + heads = 64 // tp_size + kv_heads = (16 if is_local else 8) // tp_size + head_dim = 128 + d_rel = 16 + rel_extent = 512 if is_local else 1024 + page_size = 16 + num_blocks = (tokens + page_size - 1) // page_size + q_width = heads * head_dim + kv_width = kv_heads * head_dim + r_width = heads * d_rel + device = "cuda" + + qkvr = torch.randn( + tokens, + q_width + 2 * kv_width + r_width, + device=device, + dtype=torch.bfloat16, + ) + k_weight = torch.randn(kv_width, 4, device=device, dtype=torch.bfloat16) + v_weight = torch.randn_like(k_weight) + q_norm_weight = torch.randn(head_dim, device=device, dtype=torch.bfloat16) + k_norm_weight = torch.randn_like(q_norm_weight) + rel_proj = torch.randn(d_rel, rel_extent, device=device, dtype=torch.bfloat16) + conv_cache = torch.zeros( + num_blocks, + kv_heads, + page_size, + 2 * head_dim, + device=device, + dtype=torch.bfloat16, + ) + key_cache = torch.empty( + num_blocks, + page_size, + kv_heads, + head_dim, + device=device, + dtype=torch.bfloat16, + ) + value_cache = torch.empty_like(key_cache) + positions = torch.arange(tokens, device=device, dtype=torch.int64) + block_table = torch.arange(num_blocks, device=device, dtype=torch.int32)[None] + seq_idx = torch.zeros(tokens, device=device, dtype=torch.int32) + slots = torch.arange(tokens, device=device, dtype=torch.int64) + query_start = torch.zeros(tokens, device=device, dtype=torch.int32) + log_scaling_n_floor = None if is_local else 128000 + log_scaling = None + if log_scaling_n_floor is not None: + log_scaling = torch.linspace( + 1.0, + 1.1, + tokens, + device=device, + dtype=torch.float32, + ) + return ( + qkvr, + k_weight, + v_weight, + q_norm_weight, + k_norm_weight, + rel_proj, + 1e-6, + heads, + kv_heads, + head_dim, + d_rel, + conv_cache, + key_cache, + value_cache, + positions, + block_table, + seq_idx, + slots, + query_start, + slots, + 0, + head_dim, + page_size, + log_scaling, + ) + + +def _reference(args): + qkvr = args[0] + heads, kv_heads, head_dim, d_rel = args[7:11] + q_width = heads * head_dim + kv_width = kv_heads * head_dim + q, k, v, r = qkvr.split((q_width, kv_width, kv_width, heads * d_rel), dim=1) + conv_cache = args[11].clone() + k = sconv.fused_sconv( + k.contiguous(), + args[1], + conv_cache, + args[14], + args[15], + args[16], + args[17], + args[18], + args[20], + head_dim, + args[22], + ) + v = sconv.fused_sconv( + v.contiguous(), + args[2], + conv_cache, + args[14], + args[15], + args[16], + args[17], + args[18], + args[21], + head_dim, + args[22], + ) + + def rms_norm(x, weight): + x = x.reshape(-1, head_dim).float() + rstd = torch.rsqrt(x.square().mean(1, keepdim=True) + args[6]) + return (x * rstd * weight.float()).to(qkvr.dtype) + + q = rms_norm(q, args[3]).view(qkvr.shape[0], heads, head_dim) + k = rms_norm(k, args[4]).view(qkvr.shape[0], kv_heads, head_dim) + v = v.view(qkvr.shape[0], kv_heads, head_dim) + rel = torch.mm(r.reshape(-1, d_rel), args[5]).view(qkvr.shape[0], heads, -1) + if args[23] is not None: + q = (q.float() * args[23][:, None, None]).to(q.dtype) + rel = (rel.float() * args[23][:, None, None]).to(rel.dtype) + + key_cache = args[12].clone() + value_cache = args[13].clone() + slots = args[19] + valid = slots >= 0 + key_cache.view(-1, kv_heads, head_dim)[slots[valid]] = k[valid] + value_cache.view(-1, kv_heads, head_dim)[slots[valid]] = v[valid] + return q.flatten(1), rel, key_cache, value_cache + + +@pytest.mark.skipif( + _cap is None, + reason="Inkling QKVR prep kernels require CUDA", +) +@pytest.mark.parametrize( + ("is_local", "tokens", "tp_size"), + [ + (True, 9, 4), + (False, 33, 4), + (True, 33, 8), + (True, 128, 4), + (False, 128, 8), + (True, 512, 4), + (False, 640, 8), + ], +) +@torch.inference_mode() +def test_qkvr_prep_matches_reference(is_local, tokens, tp_size): + args = _make_inputs(is_local=is_local, tokens=tokens, tp_size=tp_size) + ref_q, ref_rel, ref_key, ref_value = _reference(args) + + q, rel = qkvr_prep.fused_qkvr_prep(*args) + + torch.testing.assert_close(q, ref_q, rtol=0.01, atol=0.02) + torch.testing.assert_close(rel, ref_rel, rtol=0.02, atol=0.125) + torch.testing.assert_close(args[12], ref_key, rtol=0.01, atol=0.01) + torch.testing.assert_close(args[13], ref_value, rtol=0, atol=0) + + +@pytest.mark.skipif( + _cap is None, + reason="Inkling QKVR prep kernels require CUDA", +) +@pytest.mark.parametrize("tokens", [9, 128]) +@torch.inference_mode() +def test_qkvr_log_scaling_preserves_bf16_norm_output(tokens): + args = list(_make_inputs(is_local=False, tokens=tokens, tp_size=8)) + tau = args[23] + assert tau is not None + + scaled_q, scaled_rel = qkvr_prep.fused_qkvr_prep(*args) + args[23] = None + unscaled_q, unscaled_rel = qkvr_prep.fused_qkvr_prep(*args) + + expected_q = (unscaled_q.float() * tau[:, None]).to(unscaled_q.dtype) + expected_rel = (unscaled_rel.float() * tau[:, None, None]).to(unscaled_rel.dtype) + torch.testing.assert_close(scaled_q, expected_q, rtol=0, atol=0) + torch.testing.assert_close(scaled_rel, expected_rel, rtol=0, atol=0) + + +@pytest.mark.skipif( + _cap is None, + reason="Inkling QKVR prep kernels require CUDA", +) +@pytest.mark.parametrize("tokens", [9, 128]) +@torch.inference_mode() +def test_qkvr_prep_negative_slots_skip_cache_writes(tokens): + args = list(_make_inputs(is_local=True, tokens=tokens)) + args[17].fill_(-1) + args[19].fill_(-1) + conv_cache = args[11].clone() + key_cache = args[12].clone() + value_cache = args[13].clone() + + qkvr_prep.fused_qkvr_prep(*args) + + torch.testing.assert_close(args[11], conv_cache, rtol=0, atol=0) + torch.testing.assert_close(args[12], key_cache, rtol=0, atol=0) + torch.testing.assert_close(args[13], value_cache, rtol=0, atol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@torch.inference_mode() +def test_fused_sconv_negative_slots_skip_cache_writes(): + args = list(_make_inputs(is_local=True, tokens=9)) + args[17].fill_(-1) + cache = args[11].clone() + + sconv.fused_sconv( + args[0][:, 16 * 128 : 16 * 128 + 4 * 128].contiguous(), + args[1], + args[11], + args[14], + args[15], + args[16], + args[17], + args[18], + 0, + 128, + args[22], + ) + + torch.testing.assert_close(args[11], cache, rtol=0, atol=0) + + +@pytest.mark.parametrize( + ("rel_extent", "last_latency_rows", "first_throughput_rows"), + [(512, 8191, 8192), (1024, 2047, 2048)], +) +def test_rel_projection_schedule_crossover( + rel_extent, last_latency_rows, first_throughput_rows +): + assert not qkvr_prep.use_rel_proj_throughput(last_latency_rows, rel_extent) + assert qkvr_prep.use_rel_proj_throughput(first_throughput_rows, rel_extent) diff --git a/tests/models/inkling/test_sconv_metadata.py b/tests/models/inkling/test_sconv_metadata.py new file mode 100644 index 0000000000..70a091d382 --- /dev/null +++ b/tests/models/inkling/test_sconv_metadata.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Parity test for the fused sconv seq-metadata kernel. + +``sconv_seq_metadata`` fills the per-token ``seq_idx`` (owning request) and +``query_start`` (first x-row of that request) buffers in a single launch. Actual +tokens must match the searchsorted-based reference, while CUDA graph padding +must be initialized to safe zero values. +""" + +import pytest +import torch + +from vllm.models.inkling.nvidia.ops.sconv import sconv_seq_metadata +from vllm.models.inkling.nvidia.sconv_swa_attn import InklingSconvMetadataBuilder + +CASES = [ + # (query_lens, extra_pad_tokens) + ([1], 0), # bsz1 decode + ([1] * 8, 0), # uniform decode + ([1] * 8, 3), # uniform decode, padded tokens past the last request + ([2] * 4, 0), # uniform spec-decode + ([2048], 0), # single prefill + ([517, 1, 1, 33, 1, 128], 0), # mixed prefill/decode + ([517, 1, 1, 33, 1, 128], 5), # mixed, padded + ([1] * 500, 0), # many requests (deep binary search) +] + + +def _ref(query_start_loc: torch.Tensor, num_reqs: int, num_tokens: int): + cu_seqlens = query_start_loc[: num_reqs + 1].to(torch.int64) + token_idx = torch.arange(num_tokens, device=cu_seqlens.device, dtype=torch.int64) + seq_idx = (torch.searchsorted(cu_seqlens, token_idx, right=True) - 1).clamp( + max=num_reqs - 1 + ) + return seq_idx.to(torch.int32), cu_seqlens[seq_idx].to(torch.int32) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("query_lens,extra_pad", CASES) +def test_sconv_seq_metadata_matches_searchsorted(query_lens, extra_pad): + device = "cuda" + num_reqs = len(query_lens) + query_start_loc = torch.tensor( + [0] + list(torch.tensor(query_lens).cumsum(0)), dtype=torch.int32 + ).to(device) + num_actual_tokens = int(query_start_loc[-1]) + num_padded_tokens = num_actual_tokens + extra_pad + + ref_seq, ref_qs = _ref(query_start_loc, num_reqs, num_actual_tokens) + + seq_idx = torch.full((num_padded_tokens,), -1, dtype=torch.int32, device=device) + query_start = torch.full_like(seq_idx, -1) + sconv_seq_metadata( + query_start_loc, + num_reqs, + num_actual_tokens, + seq_idx, + query_start, + num_padded_tokens, + ) + + torch.testing.assert_close(seq_idx[:num_actual_tokens], ref_seq, rtol=0, atol=0) + torch.testing.assert_close(query_start[:num_actual_tokens], ref_qs, rtol=0, atol=0) + assert torch.count_nonzero(seq_idx[num_actual_tokens:]) == 0 + assert torch.count_nonzero(query_start[num_actual_tokens:]) == 0 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_sconv_metadata_reuses_padded_static_buffers(): + device = torch.device("cuda") + builder = object.__new__(InklingSconvMetadataBuilder) + builder.seq_idx_buffer = torch.empty(8, dtype=torch.int32, device=device) + builder.query_start_buffer = torch.empty(8, dtype=torch.int32, device=device) + + class CommonMetadata: + num_reqs = 1 + num_actual_tokens = 8 + query_start_loc = torch.tensor([0, 3], dtype=torch.int32, device=device) + query_start_loc_cpu = torch.tensor([0, 3], dtype=torch.int32) + block_table_tensor = torch.zeros((1, 1), dtype=torch.int32, device=device) + slot_mapping = torch.tensor( + [0, 1, 2, -1, -1, -1, -1, -1], + dtype=torch.int64, + device=device, + ) + + common = CommonMetadata() + first = builder.build(0, common) + pointers = tuple( + tensor.data_ptr() + for tensor in ( + first.block_table, + first.slot_mapping, + first.seq_idx, + first.query_start, + ) + ) + + common.query_start_loc[1] = 5 + common.query_start_loc_cpu[1] = 5 + common.slot_mapping[:5] = torch.arange(5, dtype=torch.int64, device=device) + second = builder.build(0, common) + + assert second.slot_mapping.shape == (8,) + assert second.seq_idx.shape == (8,) + assert second.query_start.shape == (8,) + assert pointers == tuple( + tensor.data_ptr() + for tensor in ( + second.block_table, + second.slot_mapping, + second.seq_idx, + second.query_start, + ) + ) + assert torch.count_nonzero(second.seq_idx[5:]) == 0 + assert torch.count_nonzero(second.query_start[5:]) == 0 + assert torch.all(second.slot_mapping[5:] == -1) diff --git a/tests/models/registry.py b/tests/models/registry.py index cecd2d965a..d85227b75c 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -330,6 +330,12 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "naver-hyperclovax/HyperCLOVAX-SEED-Think-14B", min_transformers_version="5.9.0", ), + "InklingForCausalLM": _HfExamplesInfo( + "thinkingmachines/Inkling-NVFP4", + tokenizer_mode="inkling", + trust_remote_code=True, + max_model_len=4096, + ), "InternLM2ForCausalLM": _HfExamplesInfo( "internlm/internlm2-chat-7b", trust_remote_code=True ), @@ -956,6 +962,12 @@ _MULTIMODAL_EXAMPLE_MODELS = { "HuggingFaceM4/Idefics3-8B-Llama3", extras={"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"}, ), + "InklingForConditionalGeneration": _HfExamplesInfo( + "thinkingmachines/Inkling-NVFP4", + tokenizer_mode="inkling", + trust_remote_code=True, + max_model_len=4096, + ), "IsaacForConditionalGeneration": _HfExamplesInfo( "PerceptronAI/Isaac-0.1", trust_remote_code=True, diff --git a/tests/parser/engine/test_delegating_replay.py b/tests/parser/engine/test_delegating_replay.py index 1f2086fda3..b922e71e39 100644 --- a/tests/parser/engine/test_delegating_replay.py +++ b/tests/parser/engine/test_delegating_replay.py @@ -88,6 +88,11 @@ def _discover_pairings() -> list[_PairingInfo]: if cfg.name not in _BUILDERS: missing_builders.append(f"{engine_cls.__name__} (config.name={cfg.name!r})") continue + if cfg.name == "inkling": + # Inkling uses typed structural blocks and opts out of token-id + # terminal matching; combined-parser replay coverage lives in + # test_inkling.py. + continue parser_cls = type( f"_Delegating{engine_cls.__name__}", diff --git a/tests/parser/engine/test_inkling.py b/tests/parser/engine/test_inkling.py new file mode 100644 index 0000000000..c0b9877db3 --- /dev/null +++ b/tests/parser/engine/test_inkling.py @@ -0,0 +1,506 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the engine-based Inkling parser. + +Inkling output is a sequence of typed content blocks delimited by dedicated +special tokens; the tool-call payload is ``{"name":...,"args":{...}}`` +between ``<|content_invoke_tool_json|>`` and ``<|end_message|>``. The +cases mirror the Rust unified parser's tests +(``rust/src/parser/src/unified/inkling.rs``) where applicable. +""" + +import json + +import pytest + +from tests.parser.engine.conftest import make_mock_tokenizer +from tests.parser.engine.streaming_helpers import ( + collect_content, + collect_function_name, + collect_tool_arguments, +) +from vllm.parser.engine.parser_engine_config import ParserState +from vllm.parser.inkling import InklingParser, _inkling_arg_converter +from vllm.parser.parser_manager import ParserManager + +MSG_MODEL = "<|message_model|>" +TEXT_START = "<|content_text|>" +THINK_START = "<|content_thinking|>" +TOOL_JSON = "<|content_invoke_tool_json|>" +TOOL_TEXT = "<|content_invoke_tool_text|>" +TOOL_ERROR = "<|content_tool_error|>" +END_MESSAGE = "<|end_message|>" +END_SAMPLING = "<|content_model_end_sampling|>" + +_TML_VOCAB = { + MSG_MODEL: 200001, + TEXT_START: 200004, + END_SAMPLING: 200006, + THINK_START: 200008, + END_MESSAGE: 200010, + TOOL_ERROR: 200022, + TOOL_JSON: 200049, + TOOL_TEXT: 200057, +} + + +@pytest.fixture +def mock_tokenizer(): + return make_mock_tokenizer(_TML_VOCAB) + + +@pytest.fixture +def parser(mock_tokenizer): + return InklingParser(mock_tokenizer) + + +def _tool_block(name: str, args: str) -> str: + return f'{TOOL_JSON}{{"name":"{name}","args":{args}}}{END_MESSAGE}' + + +_MARKERS = sorted(_TML_VOCAB, key=len, reverse=True) + + +def _tokenize(text: str) -> list[tuple[int, str]]: + """Tokenize like the real stream: markers are atomic special tokens, + plain text becomes one token per character (matching the mock + tokenizer's ``chr``-based decode).""" + tokens: list[tuple[int, str]] = [] + i = 0 + while i < len(text): + for marker in _MARKERS: + if text.startswith(marker, i): + tokens.append((_TML_VOCAB[marker], marker)) + i += len(marker) + break + else: + tokens.append((ord(text[i]), text[i])) + i += 1 + return tokens + + +def _stream(parser, request, text: str, chunk_size: int): + """Stream production-shaped deltas: ``chunk_size`` tokens per delta, + with delta_token_ids covering every token (specials and text).""" + tokens = _tokenize(text) + results = [] + previous_text = "" + previous_token_ids: list[int] = [] + for start in range(0, len(tokens), chunk_size): + batch = tokens[start : start + chunk_size] + delta_text = "".join(t for _, t in batch) + delta_token_ids = [tid for tid, _ in batch] + current_text = previous_text + delta_text + current_token_ids = previous_token_ids + delta_token_ids + delta = parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=delta_text, + previous_token_ids=tuple(previous_token_ids), + current_token_ids=tuple(current_token_ids), + delta_token_ids=tuple(delta_token_ids), + request=request, + ) + results.append((delta, current_text)) + previous_text = current_text + previous_token_ids = current_token_ids + finish = parser.finish_streaming() + if finish is not None: + results.append((finish, text)) + return results + + +def _stream_text_only(parser, request, text: str, chunk_size: int): + """Stream text-only deltas (no token ids), chunked at arbitrary + character boundaries — exercises the text-lexing fallback path, + including markers split across chunks.""" + results = [] + previous_text = "" + for start in range(0, len(text), chunk_size): + delta_text = text[start : start + chunk_size] + current_text = previous_text + delta_text + delta = parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=delta_text, + previous_token_ids=(), + current_token_ids=(), + delta_token_ids=(), + request=request, + ) + results.append((delta, current_text)) + previous_text = current_text + finish = parser.finish_streaming() + if finish is not None: + results.append((finish, text)) + return results + + +def _collect_reasoning(results) -> str: + return "".join(d.reasoning for d, _ in results if d and d.reasoning) + + +class TestArgConverter: + def test_complete_wrapper(self): + raw = '{"name":"get_weather","args":{"city":"SF"}}' + assert _inkling_arg_converter(raw, False) == '{"city":"SF"}' + + def test_partial_before_args(self): + assert _inkling_arg_converter('{"name":"get_w', True) == "" + + def test_partial_inside_args(self): + raw = '{"name":"x","args":{"a":1' + assert _inkling_arg_converter(raw, True) == '{"a":1' + + def test_prefix_stability(self): + full = '{"name":"x","args":{"a":{"b":[1,2]},"c":"d"}}' + prev = "" + for end in range(len(full)): + out = _inkling_arg_converter(full[:end], True) + assert out.startswith(prev) or prev.startswith(out) or not prev + if out.startswith(prev): + prev = out + + def test_args_value_appearing_in_name(self): + raw = '{"name":"args","args":{"k":1}}' + assert _inkling_arg_converter(raw, False) == '{"k":1}' + + def test_whitespace_tolerated(self): + raw = '{ "name" : "x" , "args" : {"a": 1} }' + assert _inkling_arg_converter(raw, False) == '{"a": 1}' + + def test_missing_args_defaults_empty(self): + assert _inkling_arg_converter('{"name":"x"}', False) == "{}" + + def test_non_object_args_rejected(self): + with pytest.raises(ValueError, match="JSON object"): + _inkling_arg_converter('{"name":"x","args":[1]}', False) + + +class TestNonStreaming: + def test_plain_text(self, parser, mock_request): + reasoning, content, tools = parser.parse( + f"{TEXT_START}hello world{END_MESSAGE}", mock_request + ) + assert reasoning is None + assert content == "hello world" + assert tools is None + + def test_reasoning_text_tool(self, parser, mock_request): + text = ( + f"{THINK_START}I should check the weather.{END_MESSAGE}" + f"{MSG_MODEL}{TEXT_START}Let me check.{END_MESSAGE}" + f"{MSG_MODEL}" + _tool_block("get_weather", '{"city":"SF"}') + ) + reasoning, content, tools = parser.parse(text, mock_request) + assert reasoning == "I should check the weather." + assert content == "Let me check." + assert [t.name for t in tools] == ["get_weather"] + assert json.loads(tools[0].arguments) == {"city": "SF"} + + def test_tool_header_name_is_not_visible_content(self, parser, mock_request): + text = "get_weather" + _tool_block("get_weather", '{"city":"SF"}') + _, content, tools = parser.parse(text, mock_request) + assert content is None + assert [tool.name for tool in tools] == ["get_weather"] + + def test_parallel_tool_calls(self, parser, mock_request): + text = _tool_block("a", "{}") + MSG_MODEL + _tool_block("b", '{"x":[1,2]}') + _, _, tools = parser.parse(text, mock_request) + assert [t.name for t in tools] == ["a", "b"] + assert json.loads(tools[0].arguments) == {} + assert json.loads(tools[1].arguments) == {"x": [1, 2]} + + def test_nested_args(self, parser, mock_request): + args = '{"q":{"deep":{"list":[{"k":"v"}]}},"s":"a}b"}' + _, _, tools = parser.parse(_tool_block("f", args), mock_request) + assert json.loads(tools[0].arguments) == json.loads(args) + + def test_invoke_tool_text_is_visible_text(self, parser, mock_request): + reasoning, content, tools = parser.parse( + f"{TOOL_TEXT}do something{END_MESSAGE}", mock_request + ) + assert content == "do something" + assert tools is None + + def test_tool_error_is_visible_text(self, parser, mock_request): + _, content, tools = parser.parse(f"{TOOL_ERROR}boom{END_MESSAGE}", mock_request) + assert content == "boom" + assert tools is None + + def test_end_sampling_closes_blocks(self, parser, mock_request): + reasoning, content, _ = parser.parse( + f"{THINK_START}hm{END_MESSAGE}{MSG_MODEL}{TEXT_START}hi{END_SAMPLING}", + mock_request, + ) + assert reasoning == "hm" + assert content == "hi" + + def test_multiple_reasoning_blocks_concatenate(self, parser, mock_request): + text = ( + f"{THINK_START}one{END_MESSAGE}" + f"{MSG_MODEL}{TEXT_START}mid{END_MESSAGE}" + f"{MSG_MODEL}{THINK_START}two{END_MESSAGE}" + ) + reasoning, content, _ = parser.parse(text, mock_request) + assert reasoning == "onetwo" + assert content == "mid" + + def test_text_after_tool_call(self, parser, mock_request): + text = _tool_block("f", "{}") + f"{MSG_MODEL}{TEXT_START}done{END_MESSAGE}" + _, content, tools = parser.parse(text, mock_request) + assert [t.name for t in tools] == ["f"] + assert content == "done" + + def test_incomplete_tool_call_at_eos(self, parser, mock_request): + # Engine convention: best-effort with what arrived. (The Rust + # parser instead errors with "incomplete Inkling tool call".) + _, _, tools = parser.parse( + f'{TOOL_JSON}{{"name":"d","args":{{"k":"v"', mock_request + ) + assert [t.name for t in tools] == ["d"] + + def test_prose_marker_without_token_ids_is_structural(self, parser, mock_request): + # Inkling opts into text-lexer terminal recognition so held-back + # structural marker text from the detokenizer is still parsed. + _, content, _ = parser.parse( + f"{TEXT_START}see {TEXT_START} token{END_MESSAGE}", mock_request + ) + assert content == "see token" + + +class TestStreaming: + @pytest.mark.parametrize("chunk_size", [1, 3, 7, 64, 4096]) + def test_chunk_invariance_tool_call(self, mock_tokenizer, mock_request, chunk_size): + parser = InklingParser(mock_tokenizer) + text = f"{TEXT_START}Check this.{END_MESSAGE}{MSG_MODEL}" + _tool_block( + "get_weather", '{"city":"San Francisco"}' + ) + results = _stream(parser, mock_request, text, chunk_size) + assert collect_content(results) == "Check this." + assert collect_function_name(results) == "get_weather" + assert json.loads(collect_tool_arguments(results)) == {"city": "San Francisco"} + + @pytest.mark.parametrize("chunk_size", [1, 3, 7, 64]) + def test_chunk_invariance_tool_call_text_only( + self, mock_tokenizer, mock_request, chunk_size + ): + # Same case through the text-lexing fallback (no token ids), + # with markers split at arbitrary character boundaries. + parser = InklingParser(mock_tokenizer) + text = f"{TEXT_START}Check this.{END_MESSAGE}{MSG_MODEL}" + _tool_block( + "get_weather", '{"city":"San Francisco"}' + ) + results = _stream_text_only(parser, mock_request, text, chunk_size) + assert collect_content(results) == "Check this." + assert collect_function_name(results) == "get_weather" + assert json.loads(collect_tool_arguments(results)) == {"city": "San Francisco"} + + @pytest.mark.parametrize("chunk_size", [1, 5, 11]) + def test_chunk_invariance_reasoning(self, mock_tokenizer, mock_request, chunk_size): + parser = InklingParser(mock_tokenizer) + text = ( + f"{THINK_START}thinking...{END_MESSAGE}" + f"{MSG_MODEL}{TEXT_START}answer{END_MESSAGE}" + ) + results = _stream(parser, mock_request, text, chunk_size) + assert _collect_reasoning(results) == "thinking..." + assert collect_content(results) == "answer" + + def test_split_marker_held_across_chunks(self, parser, mock_request): + # Mirrors Rust `inkling_streaming_holds_split_markers`. + text = f"{TEXT_START}hello{END_MESSAGE}" + results = _stream_text_only(parser, mock_request, text, 9) + assert collect_content(results) == "hello" + + def test_name_streams_before_args_complete(self, parser, mock_request): + # Feed only up to the name's closing quote — the name delta must + # already be emitted before any args arrive. + prefix = f'{TOOL_JSON}{{"name":"get_weather",' + results = _stream(parser, mock_request, prefix, 4096) + assert collect_function_name(results) == "get_weather" + + def test_combined_parser_reasoning_to_tool_handoff_uses_text_markers( + self, mock_tokenizer, mock_request + ): + parser_cls = ParserManager.get_parser( + tool_parser_name="inkling", + reasoning_parser_name="inkling", + enable_auto_tools=True, + ) + parser = parser_cls(mock_tokenizer, []) + + first = parser.parse_delta( + THINK_START, + [_TML_VOCAB[THINK_START]], + mock_request, + prompt_token_ids=[_TML_VOCAB[MSG_MODEL]], + finished=False, + ) + assert first is None + + second = parser.parse_delta( + "thinking", + [ord(c) for c in "thinking"], + mock_request, + finished=False, + ) + assert second is not None + assert second.reasoning == "thinking" + + # Mirrors the DelegatingParser handoff after reasoning closes: the + # tool pass receives reconstructed text that starts at the Inkling + # tool marker, while the token-id slice has already moved past it. + body = ( + "get_weather" + f'{TOOL_JSON}{{"name":"get_weather","args":{{"city":"Seattle"}}}}' + f"{END_MESSAGE}" + ) + third = parser.parse_delta( + body, + [_TML_VOCAB[END_MESSAGE], _TML_VOCAB[END_SAMPLING]], + mock_request, + finished=True, + ) + assert third is not None + assert third.tool_calls + assert third.tool_calls[0].function.name == "get_weather" + assert third.tool_calls[0].function.arguments == '{"city":"Seattle"}' + assert TOOL_JSON not in ((third.content or "") + (third.reasoning or "")) + + def test_streamed_args_are_object_only(self, parser, mock_request): + # The streamed `arguments` must be the bare args object, never + # the `{"name":...}` wrapper. + text = _tool_block("f", '{"a":1}') + results = _stream(parser, mock_request, text, 3) + args = collect_tool_arguments(results) + assert json.loads(args) == {"a": 1} + assert "name" not in args + + @pytest.mark.parametrize("chunk_size", [1, 9]) + def test_parallel_calls_streaming(self, mock_tokenizer, mock_request, chunk_size): + parser = InklingParser(mock_tokenizer) + text = _tool_block("a", '{"i":1}') + MSG_MODEL + _tool_block("b", '{"i":2}') + results = _stream(parser, mock_request, text, chunk_size) + indexed: dict[int, dict[str, str]] = {} + for delta, _ in results: + if not (delta and delta.tool_calls): + continue + for tc in delta.tool_calls: + slot = indexed.setdefault(tc.index, {"name": "", "args": ""}) + if tc.function and tc.function.name: + slot["name"] = tc.function.name + if tc.function and tc.function.arguments: + slot["args"] += tc.function.arguments + assert indexed[0]["name"] == "a" + assert indexed[1]["name"] == "b" + assert json.loads(indexed[0]["args"]) == {"i": 1} + assert json.loads(indexed[1]["args"]) == {"i": 2} + + +class TestPromptSeededState: + def test_prompt_ending_in_thinking_starts_reasoning(self, parser, mock_request): + parser.adjust_initial_state_from_prompt([200001, _TML_VOCAB[THINK_START]]) + assert parser._engine.state == ParserState.REASONING + + def test_prompt_ending_in_text_starts_content(self, parser): + parser.adjust_initial_state_from_prompt([200001, _TML_VOCAB[TEXT_START]]) + assert parser._engine.state == ParserState.CONTENT + + def test_generation_prompt_tail_starts_message_header(self, parser): + parser.adjust_initial_state_from_prompt( + [_TML_VOCAB[END_MESSAGE], _TML_VOCAB[MSG_MODEL]] + ) + assert parser._engine.state == ParserState.MESSAGE_HEADER + + def test_generation_prompt_header_hides_tool_name(self, parser, mock_request): + text = "get_weather" + _tool_block("get_weather", '{"city":"SF"}') + delta = parser.parse_delta( + text, + [token_id for token_id, _ in _tokenize(text)], + mock_request, + prompt_token_ids=[_TML_VOCAB[END_MESSAGE], _TML_VOCAB[MSG_MODEL]], + finished=True, + ) + assert delta is not None + assert delta.content is None + assert delta.tool_calls[0].function.name == "get_weather" + + +class TestToolCallFiltering: + """Inkling equivalents of the generic tool-call-filtering replay tests + (Inkling is excluded from those in test_replay.py: its structural + role/kind tokens and shared block-end token don't fit the generic + reasoning/tool split model).""" + + def test_skip_tool_parsing_round_trip(self, mock_tokenizer, mock_request): + # First pass (reasoning adapter, skip_tool_parsing): reasoning is + # classified as reasoning while tool markup survives in content; + # second pass (tool adapter) re-extracts the calls from it. + text = ( + f"{THINK_START}plan{END_MESSAGE}{MSG_MODEL}" + + _tool_block("f", '{"a":1}') + + MSG_MODEL + + _tool_block("g", '{"b":[2]}') + ) + first = InklingParser(mock_tokenizer) + first.skip_tool_parsing = True + reasoning, content = first.extract_reasoning(text, mock_request) + assert reasoning == "plan" + assert content.count(TOOL_JSON) == 2 + + second = InklingParser(mock_tokenizer) + result = second.extract_tool_calls_from_content(content, mock_request) + assert result.tools_called + assert [tc.function.name for tc in result.tool_calls] == ["f", "g"] + assert json.loads(result.tool_calls[0].function.arguments) == {"a": 1} + assert json.loads(result.tool_calls[1].function.arguments) == {"b": [2]} + + @pytest.fixture + def none_request(self, mock_request): + mock_request.tools = [{"type": "function", "function": {"name": "f"}}] + mock_request.tool_choice = "none" + return mock_request + + def test_tool_choice_none_non_streaming(self, mock_tokenizer, none_request): + parser = InklingParser(mock_tokenizer) + text = ( + f"{THINK_START}plan{END_MESSAGE}" + f"{MSG_MODEL}{TEXT_START}visible{END_MESSAGE}" + f"{MSG_MODEL}" + _tool_block("f", '{"a":1}') + ) + reasoning, content, tools = parser.parse(text, none_request) + assert reasoning == "plan" + assert content == "visible" + assert not tools + + def test_tool_choice_none_streaming(self, mock_tokenizer, none_request): + parser = InklingParser(mock_tokenizer) + text = f"{TEXT_START}visible{END_MESSAGE}{MSG_MODEL}" + _tool_block( + "f", '{"a":1}' + ) + results = _stream(parser, none_request, text, 3) + assert collect_content(results) == "visible" + assert all(not (d and d.tool_calls) for d, _ in results) + + +class TestRegisteredAdapters: + def test_adapters_resolve(self): + from vllm.reasoning import ReasoningParserManager + from vllm.tool_parsers import ToolParserManager + + reasoning_cls = ReasoningParserManager.get_reasoning_parser("inkling") + tool_cls = ToolParserManager.get_tool_parser("inkling") + assert reasoning_cls._parser_engine_cls is InklingParser + assert tool_cls._parser_engine_cls is InklingParser + assert tool_cls.supports_required_and_named is False + + def test_adapter_round_trip(self, mock_tokenizer, mock_request): + from vllm.tool_parsers import ToolParserManager + + tool_cls = ToolParserManager.get_tool_parser("inkling") + adapter = tool_cls(mock_tokenizer) + result = adapter.extract_tool_calls(_tool_block("f", '{"a":1}'), mock_request) + assert result.tools_called + assert result.tool_calls[0].function.name == "f" + assert json.loads(result.tool_calls[0].function.arguments) == {"a": 1} diff --git a/tests/parser/engine/test_replay.py b/tests/parser/engine/test_replay.py index 1626135f78..064e1b6236 100644 --- a/tests/parser/engine/test_replay.py +++ b/tests/parser/engine/test_replay.py @@ -68,6 +68,10 @@ def _discover_parsers() -> list[_ParserInfo]: if cfg.name not in _BUILDERS: missing_builders.append(f"{obj.__name__} (config.name={cfg.name!r})") continue + if cfg.name == "inkling": + # Inkling opts out of token-id terminal matching and has typed + # structural blocks; its replay coverage lives in test_inkling.py. + continue tool_end = cfg.token_id_terminals.get("TOOL_END") if not tool_end: raise RuntimeError( @@ -299,7 +303,12 @@ _TOOL_CALL_SAMPLES = [ (p.parser_cls, s, p.think_end, p.tool_start) for p in _PARSERS for s in p.samples - if s.expected_tool_calls and s.expected_reasoning + if s.expected_tool_calls + and s.expected_reasoning + # Inkling's typed-block format (structural role/kind tokens, shared + # block-end token) doesn't fit the generic reasoning/tool split + # below; its filtering modes are covered in test_inkling.py instead. + and p.name != "inkling" ] diff --git a/tests/parser/engine/trace_builder.py b/tests/parser/engine/trace_builder.py index 533c64408f..f24e3318f9 100644 --- a/tests/parser/engine/trace_builder.py +++ b/tests/parser/engine/trace_builder.py @@ -33,6 +33,7 @@ from vllm.parser.engine.registered_adapters import ( DeepSeekV32Parser, Gemma4Parser, Glm47MoeParser, + InklingParser, KimiK2Parser, MinimaxM2Parser, NemotronV3Parser, @@ -948,6 +949,77 @@ _KIMI_K2_SCENARIOS = [ ] +# ── Inkling (typed content blocks, JSON tool payloads) ─────────────────── + +_TML_VOCAB: dict[str, int] = { + "<|message_model|>": 200001, + "<|content_text|>": 200004, + "<|content_model_end_sampling|>": 200006, + "<|content_thinking|>": 200008, + "<|end_message|>": 200010, + "<|content_tool_error|>": 200022, + "<|content_invoke_tool_json|>": 200049, + "<|content_invoke_tool_text|>": 200057, +} + + +def _inkling_block( + segs: list[tuple[str, bool]], + kind_token: str, + body: str, +) -> None: + """Append one Inkling content block; blocks after the first start with + the ``<|message_model|>`` role token (the first block continues the + generation prompt directly).""" + if segs: + segs.append(("<|message_model|>", True)) + segs.append((kind_token, True)) + if body: + segs.append((body, False)) + segs.append(("<|end_message|>", True)) + + +def _inkling_segments(scenario: Scenario) -> list[tuple[str, bool]]: + segs: list[tuple[str, bool]] = [] + if scenario.reasoning is not None: + _inkling_block(segs, "<|content_thinking|>", scenario.reasoning) + if scenario.tool_calls is not None and not scenario.tool_calls: + _inkling_block(segs, "<|content_invoke_tool_json|>", "") + if scenario.content is not None: + _inkling_block(segs, "<|content_text|>", scenario.content) + if scenario.tool_calls: + for tc in scenario.tool_calls: + args = json.dumps(tc.arguments, ensure_ascii=False, separators=(",", ":")) + payload = f'{{"name":"{tc.name}","args":{args}}}' + _inkling_block(segs, "<|content_invoke_tool_json|>", payload) + return segs + + +def _build_inkling(scenario: Scenario, validate: bool = True) -> Sample: + prompt_token_ids = None + if scenario.after_tool_response: + # Prompt ends with a closed tool-response block and the + # generation-prompt role token. + prompt_token_ids = [ + _TML_VOCAB["<|end_message|>"], + _TML_VOCAB["<|message_model|>"], + ] + sample = _make_sample( + sample_id=f"inkling-{scenario.id}", + description=scenario.description, + vocab=_TML_VOCAB, + segments=_inkling_segments(scenario), + expected_reasoning=scenario.reasoning, + expected_content=_qwen3_expected_content(scenario), + expected_tool_calls=_expected_tc(scenario), + tools=_expected_tools(scenario), + prompt_token_ids=prompt_token_ids, + ) + if validate: + _validate_sample(sample, InklingParser) + return sample + + # ── Registry and public API ────────────────────────────────────────── _BUILDERS: dict[str, Any] = { @@ -960,6 +1032,7 @@ _BUILDERS: dict[str, Any] = { "glm47_moe": _build_glm47_moe, "kimi_k2": _build_kimi_k2, "qwen3": _build_qwen3, + "inkling": _build_inkling, } diff --git a/tests/renderers/test_inkling.py b/tests/renderers/test_inkling.py new file mode 100644 index 0000000000..2753417b42 --- /dev/null +++ b/tests/renderers/test_inkling.py @@ -0,0 +1,484 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Golden tests for the native Inkling renderer encoding. + +The message fixtures mirror the Rust renderer's fixture tests +(``rust/src/chat/src/renderer/inkling/tests.rs``) so the two frontends stay +in token-level parity. Image blocks emit the bare ``<|content_image|>`` marker; +``InklingMultiModalProcessor`` inserts the per-patch placeholder run. +""" + +import pytest + +from vllm.renderers.inkling import ( + InklingRenderer, + _HfBackedTmlTokenizer, + _resolve_reasoning_effort, +) +from vllm.renderers.inkling_encoding import ( + SPECIAL_TOKEN_SPELLINGS, + render_inkling_messages, +) +from vllm.renderers.params import ChatParams + + +@pytest.fixture() +def should_do_global_cleanup_after_test() -> bool: + # These tests touch no distributed or device state; the global + # cleanup fixture is unnecessary (and trips a torch MPS allocator + # assert on macOS dev machines). + return False + + +# One id per special token, mirroring the real Inkling vocab layout; plain +# text encodes one token per character so decoded output is exact. +_SPECIAL_VOCAB = { + "<|message_user|>": 200000, + "<|message_model|>": 200001, + "<|message_system|>": 200002, + "<|message_tool|>": 200003, + "<|content_text|>": 200004, + "<|content_image|>": 200005, + "<|content_model_end_sampling|>": 200006, + "<|content_thinking|>": 200008, + "<|end_message|>": 200010, + "<|content_audio_input|>": 200020, + # The HF vocab spells CONTENT_XML as an unused slot. + "<|unused_200024|>": 200024, + "<|audio_end|>": 200043, + "<|content_invoke_tool_json|>": 200049, +} + +_ID_TO_SPECIAL = {v: k for k, v in _SPECIAL_VOCAB.items()} + + +class FakeHfTokenizer: + def get_vocab(self): + return dict(_SPECIAL_VOCAB) + + def encode(self, text, add_special_tokens=False): + assert not add_special_tokens + return [ord(ch) for ch in text] + + +def decode(token_ids): + return "".join( + _ID_TO_SPECIAL.get(tid, chr(tid) if tid < 200000 else f"<{tid}>") + for tid in token_ids + ) + + +@pytest.fixture +def inkling_tokenizer(): + return _HfBackedTmlTokenizer(FakeHfTokenizer()) + + +def render_text(inkling_tokenizer, messages, **kwargs): + return decode(render_inkling_messages(messages, inkling_tokenizer, **kwargs)) + + +class TestRustFixtureParity: + def test_tool_round_trip(self, inkling_tokenizer): + messages = [ + { + "role": "assistant", + "reasoning_content": "think", + "content": "answer", + "tool_calls": [ + { + "id": "call_1", + "function": { + "name": "get_weather", + "arguments": '{"city":"SF"}', + }, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + ] + assert render_text( + inkling_tokenizer, messages, add_generation_prompt=False + ) == ( + "<|message_model|><|content_thinking|>think<|end_message|>" + "<|message_model|><|content_text|>answer<|end_message|>" + "<|message_model|>get_weather<|content_invoke_tool_json|>" + '{"name":"get_weather","args":{"city":"SF"}}<|end_message|>' + "<|content_model_end_sampling|>" + "<|message_tool|>get_weather<|content_text|>sunny<|end_message|>" + ) + + def test_tool_declare(self, inkling_tokenizer): + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "required": ["city"], + "properties": {"city": {"type": "string"}}, + }, + }, + } + ] + messages = [ + { + "role": "developer", + "content": "rules", + "tools": [ + { + "type": "function", + "function": { + "name": "local_tool", + "parameters": {"z": 1, "a": {"b": 2}}, + }, + } + ], + }, + {"role": "user", "content": "hi"}, + ] + assert render_text(inkling_tokenizer, messages, tools=tools) == ( + "<|message_system|>tool_declare<|unused_200024|>" + '[{"description":"Get weather information","name":"get_weather",' + '"parameters":{"properties":{"city":{"type":"string"}},' + '"required":["city"],"type":"object"},"type":"function"},' + '{"description":"","name":"local_tool",' + '"parameters":{"a":{"b":2},"z":1},"type":"function"}]' + "<|end_message|>" + "<|message_system|><|content_text|>rules<|end_message|>" + "<|message_user|><|content_text|>hi<|end_message|>" + "<|message_model|>" + ) + + def test_text_image(self, inkling_tokenizer): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + {"type": "image_url", "image_url": "data:image/png;base64,"}, + ], + } + ] + # Multimodal preprocessing expands this bare marker after rendering. + assert render_text(inkling_tokenizer, messages) == ( + "<|message_user|><|content_text|>look<|end_message|>" + "<|message_user|><|content_image|><|end_message|>" + "<|message_model|>" + ) + + +class TestRenderingSemantics: + @pytest.mark.parametrize( + ("value", "expected"), + [ + ("none", 0.0), + ("minimal", 0.1), + ("low", 0.2), + ("medium", 0.7), + ("high", 0.9), + ("xhigh", 0.99), + ("max", 0.99), + (None, 0.9), + (0.8, 0.8), + (True, None), + ("invalid", None), + ], + ) + def test_resolve_reasoning_effort(self, value, expected): + assert _resolve_reasoning_effort(value) == expected + + def test_generation_prompt_default(self, inkling_tokenizer): + text = render_text(inkling_tokenizer, [{"role": "user", "content": "hi"}]) + assert text.endswith("<|end_message|><|message_model|>") + + def test_developer_folds_into_system(self, inkling_tokenizer): + assert ( + render_text( + inkling_tokenizer, + [{"role": "developer", "content": "be nice"}], + add_generation_prompt=False, + ) + == "<|message_system|><|content_text|>be nice<|end_message|>" + ) + + def test_empty_string_content_skipped(self, inkling_tokenizer): + assert ( + render_text( + inkling_tokenizer, + [{"role": "user", "content": ""}], + add_generation_prompt=False, + ) + == "" + ) + + def test_empty_reasoning_skipped(self, inkling_tokenizer): + assert ( + render_text( + inkling_tokenizer, + [{"role": "assistant", "reasoning_content": "", "content": "hi"}], + add_generation_prompt=False, + ) + == "<|message_model|><|content_text|>hi<|end_message|>" + "<|content_model_end_sampling|>" + ) + + def test_reasoning_field(self, inkling_tokenizer): + messages = [ + { + "role": "assistant", + "reasoning": "think", + "content": "answer", + } + ] + assert render_text( + inkling_tokenizer, messages, add_generation_prompt=False + ) == ( + "<|message_model|><|content_thinking|>think<|end_message|>" + "<|message_model|><|content_text|>answer<|end_message|>" + "<|content_model_end_sampling|>" + ) + + def test_audio_part(self, inkling_tokenizer): + messages = [ + { + "role": "user", + "content": [{"type": "input_audio", "input_audio": {}}], + } + ] + assert render_text( + inkling_tokenizer, messages, add_generation_prompt=False + ) == ("<|message_user|><|content_audio_input|><|audio_end|><|end_message|>") + + def test_tool_response_name_from_message(self, inkling_tokenizer): + assert ( + render_text( + inkling_tokenizer, + [{"role": "tool", "name": "my_tool", "content": "ok"}], + add_generation_prompt=False, + ) + == "<|message_tool|>my_tool<|content_text|>ok<|end_message|>" + ) + + def test_tool_call_args_object_form(self, inkling_tokenizer): + messages = [ + { + "role": "assistant", + "tool_calls": [ + { + "id": "x", + "function": { + "name": "f", + # dict-form arguments, unsorted keys + "arguments": {"z": 1, "a": 2}, + }, + } + ], + } + ] + assert render_text( + inkling_tokenizer, messages, add_generation_prompt=False + ) == ( + "<|message_model|>f<|content_invoke_tool_json|>" + '{"name":"f","args":{"a":2,"z":1}}<|end_message|>' + "<|content_model_end_sampling|>" + ) + + def test_tool_call_empty_args(self, inkling_tokenizer): + messages = [ + { + "role": "assistant", + "tool_calls": [{"id": "x", "function": {"name": "f", "arguments": ""}}], + } + ] + assert render_text( + inkling_tokenizer, messages, add_generation_prompt=False + ) == ( + "<|message_model|>f<|content_invoke_tool_json|>" + '{"name":"f","args":{}}<|end_message|>' + "<|content_model_end_sampling|>" + ) + + def test_tool_call_non_object_args_rejected(self, inkling_tokenizer): + messages = [ + { + "role": "assistant", + "tool_calls": [ + {"id": "x", "function": {"name": "f", "arguments": "[1]"}} + ], + } + ] + with pytest.raises(TypeError, match="decode to an object"): + render_inkling_messages( + messages, inkling_tokenizer, add_generation_prompt=False + ) + + def test_unsupported_role_rejected(self, inkling_tokenizer): + with pytest.raises(ValueError, match="unsupported Inkling message role"): + render_inkling_messages( + [{"role": "narrator", "content": "hi"}], inkling_tokenizer + ) + + +class TestReasoningEffort: + def test_frontend_defaults_to_high(self, inkling_tokenizer): + renderer = InklingRenderer.__new__(InklingRenderer) + renderer._inkling_tokenizer = inkling_tokenizer + + text = decode( + renderer._render( + [ + {"role": "system", "content": "rules"}, + {"role": "user", "content": "hi"}, + ], + ChatParams(), + ) + ) + + assert text.startswith( + "<|message_system|><|content_text|>rules<|end_message|>" + "<|message_system|><|content_text|>Thinking effort level: 0.9" + "<|end_message|>" + ) + + @pytest.mark.parametrize("value", ["none", 0, 0.0, -0.0]) + def test_zero_effort_has_one_canonical_spelling(self, inkling_tokenizer, value): + renderer = InklingRenderer.__new__(InklingRenderer) + renderer._inkling_tokenizer = inkling_tokenizer + + text = decode( + renderer._render( + [{"role": "user", "content": "hi"}], + ChatParams(chat_template_kwargs={"reasoning_effort": value}), + ) + ) + + assert text.startswith( + "<|message_system|><|content_text|>Thinking effort level: 0.0" + "<|end_message|>" + ) + + def test_emits_one_effort_after_initial_prefix(self, inkling_tokenizer): + effort_block = ( + "<|message_system|><|content_text|>Thinking effort level: 0.7" + "<|end_message|>" + ) + text = render_text( + inkling_tokenizer, + [ + {"role": "system", "content": "rules"}, + {"role": "developer", "content": "policy"}, + {"role": "user", "content": "user1"}, + {"role": "assistant", "content": "assistant1"}, + {"role": "user", "content": "user2"}, + ], + tools=[{"type": "function", "function": {"name": "f"}}], + reasoning_effort=0.7, + ) + + assert text.count(effort_block) == 1 + assert ( + text.index("tool_declare") + < text.index("rules") + < text.index("policy") + < text.index(effort_block) + < text.index("user1") + < text.index("assistant1") + < text.index("user2") + ) + + def test_renders_after_tool_declare(self, inkling_tokenizer): + tools = [{"type": "function", "function": {"name": "f"}}] + text = render_text( + inkling_tokenizer, + [{"role": "user", "content": "hi"}], + tools=tools, + reasoning_effort=0.8, + ) + declare_end = text.index("<|end_message|>") + len("<|end_message|>") + assert text[declare_end:].startswith( + "<|message_system|><|content_text|>Thinking effort level: 0.8" + "<|end_message|>" + ) + + @pytest.mark.parametrize( + ("value", "expected"), + [(0.2, "0.2"), (0.7, "0.7"), (0.9, "0.9"), (0.99, "0.99")], + ) + def test_formats_at_most_two_decimals(self, inkling_tokenizer, value, expected): + text = render_text( + inkling_tokenizer, + [{"role": "user", "content": "hi"}], + reasoning_effort=value, + ) + assert f"Thinking effort level: {expected}<|end_message|>" in text + + def test_absent_emits_no_block(self, inkling_tokenizer): + text = render_text(inkling_tokenizer, [{"role": "user", "content": "hi"}]) + assert "Thinking effort" not in text + + @pytest.mark.parametrize("value", [0.9900001, 1, 1.5, -0.1]) + def test_out_of_range_rejected(self, inkling_tokenizer, value): + with pytest.raises(ValueError, match="must be in"): + render_inkling_messages( + [{"role": "user", "content": "hi"}], + inkling_tokenizer, + reasoning_effort=value, + ) + + +class TestSpecialTokenResolution: + def test_missing_special_token_raises(self): + class IncompleteTokenizer(FakeHfTokenizer): + def get_vocab(self): + vocab = dict(_SPECIAL_VOCAB) + del vocab["<|content_invoke_tool_json|>"] + return vocab + + with pytest.raises(ValueError, match="missing special tokens"): + _HfBackedTmlTokenizer(IncompleteTokenizer()) + + def test_semantic_spelling_preferred(self): + class SemanticSpellingTokenizer(FakeHfTokenizer): + def get_vocab(self): + vocab = dict(_SPECIAL_VOCAB) + del vocab["<|unused_200024|>"] + vocab["<|content_xml|>"] = 200024 + return vocab + + inkling_tokenizer = _HfBackedTmlTokenizer(SemanticSpellingTokenizer()) + tools = [{"type": "function", "function": {"name": "f"}}] + ids = render_inkling_messages( + [{"role": "user", "content": "hi"}], inkling_tokenizer, tools=tools + ) + assert 200024 in ids + + def test_all_spellings_covered(self): + # Every semantic token must resolve from the reference vocab. + for token, spellings in SPECIAL_TOKEN_SPELLINGS.items(): + assert any(s in _SPECIAL_VOCAB for s in spellings), token + + +def test_render_inkling_messages_direct_protocol(): + """The encoding core only needs the structural tokenizer protocol.""" + + class ProtocolTokenizer: + def encode_text(self, text): + return [ord(c) for c in text] + + def encode_special(self, token): + return { + "<|message_user|>": 200000, + "<|message_model|>": 200001, + "<|content_text|>": 200004, + "<|content_model_end_sampling|>": 200006, + "<|end_message|>": 200010, + }[token] + + ids = render_inkling_messages( + [{"role": "user", "content": "hi"}], + ProtocolTokenizer(), + add_generation_prompt=True, + ) + assert ids == [200000, 200004, ord("h"), ord("i"), 200010, 200001] diff --git a/tests/test_config.py b/tests/test_config.py index 6f2be35ae8..a785297997 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -211,6 +211,26 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1( ), True, ), + ( + SimpleNamespace( + model="thinkingmachines/Inkling", + architectures=["InklingForCausalLM"], + runner_type="generate", + is_moe=True, + is_quantized=False, + ), + True, + ), + ( + SimpleNamespace( + model="thinkingmachines/Inkling", + architectures=["InklingForConditionalGeneration"], + runner_type="generate", + is_moe=True, + is_quantized=False, + ), + True, + ), ( SimpleNamespace( model="mistralai/Mixtral-8x7B-Instruct-v0.1", diff --git a/vllm/config/model.py b/vllm/config/model.py index 0ce01de166..e36b672cd8 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -83,7 +83,9 @@ _REGISTERED_MODEL_CLASS_OVERRIDES: set[tuple[str, str]] = set() RunnerOption = Literal["auto", RunnerType] ConvertType = Literal["none", "embed", "classify"] ConvertOption = Literal["auto", ConvertType] -TokenizerMode = Literal["auto", "hf", "slow", "mistral", "deepseek_v32", "deepseek_v4"] +TokenizerMode = Literal[ + "auto", "hf", "slow", "mistral", "deepseek_v32", "deepseek_v4", "inkling" +] ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"] LogprobsMode = Literal[ "raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs" @@ -628,6 +630,8 @@ class ModelConfig: self.tokenizer_mode = "deepseek_v32" elif arch == "DeepseekV4ForCausalLM": self.tokenizer_mode = "deepseek_v4" + elif arch in ("InklingForCausalLM", "InklingForConditionalGeneration"): + self.tokenizer_mode = "inkling" if self.tokenizer_mode != "auto": logger.info( diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index dc01800341..4afaf30f37 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -68,9 +68,11 @@ logger = init_logger(__name__) DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset( { "DeepseekV2ForCausalLM", - "Qwen2MoeForCausalLM", "GraniteMoeForCausalLM", + "InklingForCausalLM", + "InklingForConditionalGeneration", "LongcatFlashNgramForCausalLM", + "Qwen2MoeForCausalLM", } ) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index cab03a8a6a..f020abdd98 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -159,6 +159,11 @@ _TEXT_GENERATION_MODELS = { "vllm.models.minimax_m3", "MiniMaxM3SparseForCausalLM", ), + "InklingForCausalLM": ("vllm.models.inkling", "InklingForCausalLM"), + "InklingForConditionalGeneration": ( + "vllm.models.inkling", + "InklingForConditionalGeneration", + ), "Ministral3ForCausalLM": ("mistral", "MistralForCausalLM"), "MistralForCausalLM": ("mistral", "MistralForCausalLM"), "MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"), diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index c55ad148b2..d4b9a735d2 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -43,6 +43,7 @@ if TYPE_CHECKING: logger = init_logger(__name__) _LL_BF16_WARMUP_MODEL_SHAPES: tuple[tuple[int, int], ...] = ( + (6144, 264), # Inkling (7168, 256), # DSV3 (7168, 384), # DSV4-Pro (14400, 256), # DSV4-Flash diff --git a/vllm/models/inkling/__init__.py b/vllm/models/inkling/__init__.py new file mode 100644 index 0000000000..21385d8fe7 --- /dev/null +++ b/vllm/models/inkling/__init__.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .nvidia.model import ( + InklingForCausalLM, + InklingForConditionalGeneration, + ) + +__all__ = [ + "InklingForConditionalGeneration", + "InklingForCausalLM", +] + + +def __getattr__(name: str): + if name in __all__: + from .nvidia import model + + return getattr(model, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/vllm/models/inkling/common/__init__.py b/vllm/models/inkling/common/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/inkling/common/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/inkling/common/mm_preprocess.py b/vllm/models/inkling/common/mm_preprocess.py new file mode 100644 index 0000000000..eb3c7b591f --- /dev/null +++ b/vllm/models/inkling/common/mm_preprocess.py @@ -0,0 +1,361 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling multimodal preprocessing.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from typing import Any, cast + +import numpy as np +import regex as re +import torch +from transformers.feature_extraction_utils import BatchFeature + +from vllm.config.multimodal import ( + AudioDummyOptions, + BaseDummyOptions, + ImageDummyOptions, +) +from vllm.inputs import MultiModalDataDict +from vllm.multimodal.inputs import ( + MultiModalFieldConfig, + MultiModalKwargsItems, +) +from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser +from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, + BaseMultiModalProcessor, + BaseProcessingInfo, + PromptReplacement, + PromptUpdate, + PromptUpdateDetails, +) +from vllm.transformers_utils.processors.inkling import ( + AUDIO_MARKER_ID, + AUDIO_TOKEN_ID, + IMAGE_MARKER_ID, + IMAGE_TOKEN_ID, + InklingAudioFeatureExtractor, + InklingImageProcessor, + InklingProcessor, +) + +from ..configs import InklingMMConfig + +# Maximum audio tokens accepted per clip. At the dMel rate of 20 tokens/s +# (50 ms hop) this is ~10 minutes of audio. It bounds the persistent per-request +# buffers and the encoder/memory budget; longer clips are rejected up front. +MAX_AUDIO_TOKENS = 12_000 + + +class InklingMultiModalDataParser(MultiModalDataParser): + def _parse_audio_data(self, data: Any) -> Any: + if isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 2: + raise ValueError( + "Inkling raw 2-D audio has an ambiguous channel layout. " + "Provide encoded audio or a list of mono waveforms." + ) + return super()._parse_audio_data(data) + + +def inkling_vision_enabled(config: InklingMMConfig) -> bool: + return getattr(config.vision_config, "decoder_dmodel", None) is not None + + +def inkling_audio_enabled(config: InklingMMConfig) -> bool: + return getattr(config.audio_config, "decoder_dmodel", None) is not None + + +class InklingProcessingInfo(BaseProcessingInfo): + def get_hf_config(self) -> InklingMMConfig: + return self.ctx.get_hf_config(InklingMMConfig) + + def get_hf_processor(self, **kwargs: object) -> InklingProcessor: + config = self.get_hf_config() + vision_config = config.vision_config + audio_config = config.audio_config + + image_processor = InklingImageProcessor( + patch_size=getattr(vision_config, "patch_size", None) or 40, + ) + + if inkling_audio_enabled(config): + audio_params = { + "n_mels": audio_config.n_mel_bins, + "num_dmel_bins": audio_config.mel_vocab_size, + "dmel_min_value": audio_config.dmel_min_value, + "dmel_max_value": audio_config.dmel_max_value, + } + else: + audio_params = {} + audio_extractor = InklingAudioFeatureExtractor(params=audio_params) + + return InklingProcessor( + image_processor=image_processor, + audio_feature_extractor=audio_extractor, + tokenizer=self.get_tokenizer(), + ) + + def get_data_parser(self) -> MultiModalDataParser: + # Audio inputs must be resampled to the dMel feature extractor's rate + # before `process_audios` (see InklingAudioFeatureExtractor._decode_one). + # Without a target_sr the default parser raises on any audio input. + extractor = self.get_hf_processor().audio_feature_extractor + return InklingMultiModalDataParser( + target_sr=extractor.params.sample_rate, + target_channels=1, + expected_hidden_size=self._get_expected_hidden_size(), + ) + + def get_supported_mm_limits(self) -> Mapping[str, int | None]: + config = self.get_hf_config() + limits: dict[str, int | None] = {} + if inkling_vision_enabled(config): + limits["image"] = None + if inkling_audio_enabled(config): + limits["audio"] = None + return limits + + def get_mm_max_tokens_per_item( + self, + seq_len: int, + mm_counts: Mapping[str, int], + ) -> Mapping[str, int] | None: + # Let vLLM profile dummy inputs to determine the max token counts; the + # image patch count is data-dependent, and the dummy audio is sized to + # MAX_AUDIO_TOKENS so audio is profiled/budgeted at its allowed maximum. + return None + + +class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]): + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: + # One placeholder per media item; the processor expands each into N + # copies once feature row counts are known. + num_images = mm_counts.get("image", 0) + num_audios = mm_counts.get("audio", 0) + # Use spellings the renderer would emit; tokenization is bypassed in + # _call_hf_processor (we build input_ids directly), so the exact text + # only needs to be a stable per-item marker. + return ("<|content_image|>" * num_images) + ( + "<|content_audio_input|>" * num_audios + ) + + def get_dummy_mm_data( + self, + seq_len: int, + mm_counts: Mapping[str, int], + mm_options: Mapping[str, BaseDummyOptions], + ) -> MultiModalDataDict: + config = self.info.get_hf_config() + num_images = mm_counts.get("image", 0) + num_audios = mm_counts.get("audio", 0) + + mm_data: dict[str, Any] = {} + if num_images: + patch_size = getattr(config.vision_config, "patch_size", 40) + # A square image ~4 patches wide so the dummy emits several patches. + side = patch_size * 4 + image_overrides = mm_options.get("image") + mm_data["image"] = self._get_dummy_images( + width=side, + height=side, + num_images=num_images, + overrides=cast(ImageDummyOptions | None, image_overrides), + ) + if num_audios: + # Size the dummy at the maximum allowed audio so memory/encoder + # budgeting reflects the largest clip we accept (MAX_AUDIO_TOKENS). + params = self.info.get_hf_processor().audio_feature_extractor.params + hop = int(round(params.audio_token_duration_s * params.sample_rate)) + audio_len = MAX_AUDIO_TOKENS * hop + audio_overrides = mm_options.get("audio") + mm_data["audio"] = self._get_dummy_audios( + length=audio_len, + num_audios=num_audios, + overrides=cast(AudioDummyOptions | None, audio_overrides), + ) + return mm_data + + +class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]): + def _hf_processor_applies_updates( + self, + prompt_text: str, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + tokenization_kwargs: Mapping[str, object], + ) -> bool: + return False + + def _call_hf_processor( + self, + prompt: str, + mm_data: Mapping[str, object], + mm_kwargs: Mapping[str, object], + tok_kwargs: Mapping[str, object], + ) -> BatchFeature: + # Inkling is not a standard HF processor (no fused text+mm call), so we run + # the vendored extractors ourselves and tokenize the text separately. + # The MM placeholders in `prompt` are expanded later by the prompt + # updates, so here we emit ONE placeholder id per media item. + processor = self.info.get_hf_processor(**mm_kwargs) + tokenizer = self.info.get_tokenizer() + + images = mm_data.get("images") or [] + audios = mm_data.get("audios") or [] + if not isinstance(images, list): + images = list(cast(Iterable[Any], images)) + if not isinstance(audios, list): + audios = list(cast(Iterable[Any], audios)) + + prompt_ids = self._tokenize_with_placeholders( + prompt, tokenizer, len(images), len(audios) + ) + + data: dict[str, Any] = {"input_ids": [prompt_ids]} + + if images: + img_feat = processor.process_images(images) + data["pixel_values"] = img_feat["vision_patches_bthwc"] + data["num_patches"] = torch.tensor( + img_feat["num_patches"], dtype=torch.int64 + ) + + if audios: + aud_feat = processor.process_audios(audios) + per_clip = aud_feat["dmel_bins"] + num_audio_tokens = aud_feat["num_audio_tokens"] + for i, n in enumerate(num_audio_tokens): + if int(n) > MAX_AUDIO_TOKENS: + raise ValueError( + f"Audio clip {i} produces {int(n)} tokens, exceeding the " + f"maximum of {MAX_AUDIO_TOKENS} (~10 min at 20 tokens/s). " + "Provide a shorter clip." + ) + if per_clip: + input_audio_features = torch.cat( + [torch.as_tensor(c) for c in per_clip], dim=0 + ) + else: + input_audio_features = torch.empty(0) + data["input_audio_features"] = input_audio_features + data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64) + + return BatchFeature(data=data, tensor_type=None) + + def _tokenize_with_placeholders( + self, + prompt: str, + tokenizer: Any, + num_images: int, + num_audios: int, + ) -> list[int]: + """Tokenize `prompt`, emitting the block-start marker id per media item. + + Each marker (kept verbatim) is later expanded by ``_get_prompt_updates`` + into `` + * N``. + """ + image_marker = "<|content_image|>" + audio_marker = "<|content_audio_input|>" + + pattern = f"({re.escape(image_marker)}|{re.escape(audio_marker)})" + chunks = re.split(pattern, prompt) + + ids: list[int] = [] + seen_img = seen_aud = 0 + for chunk in chunks: + if chunk == image_marker: + ids.append(IMAGE_MARKER_ID) + seen_img += 1 + elif chunk == audio_marker: + ids.append(AUDIO_MARKER_ID) + seen_aud += 1 + elif chunk: + ids.extend(tokenizer.encode(chunk, add_special_tokens=False)) + + # Reconcile against the declared media counts only when media is + # present. With no media items -- e.g. the base text-only tokenization + # probe (``_apply_hf_processor_text_only``), which calls this via + # ``_call_hf_processor`` with empty ``mm_data`` -- emit the markers + # verbatim; the marker<->item correspondence is enforced later by + # ``_get_prompt_updates`` once the media features are available. + if num_images or num_audios: + # Fail clearly on a placeholder/media-count mismatch instead of + # crashing with an IndexError deep in the per-item replacement logic. + if num_images and seen_img != num_images: + raise ValueError( + f"Prompt contains {seen_img} image placeholder(s), but only " + f"{num_images} image(s) were provided." + ) + if num_audios and seen_aud != num_audios: + raise ValueError( + f"Prompt contains {seen_aud} audio placeholder(s), but only " + f"{num_audios} audio input(s) were provided." + ) + return ids + + def _get_mm_fields_config( + self, + hf_inputs: BatchFeature, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> Mapping[str, MultiModalFieldConfig]: + num_patches = hf_inputs.get("num_patches", torch.empty(0, dtype=torch.int64)) + num_audio_tokens = hf_inputs.get( + "num_audio_tokens", torch.empty(0, dtype=torch.int64) + ) + return dict( + # Ragged per-image patches, grouped by num_patches. + pixel_values=MultiModalFieldConfig.flat_from_sizes("image", num_patches), + num_patches=MultiModalFieldConfig.batched("image"), + # Ragged per-audio frames, grouped by num_audio_tokens. + input_audio_features=MultiModalFieldConfig.flat_from_sizes( + "audio", num_audio_tokens + ), + num_audio_tokens=MultiModalFieldConfig.batched("audio"), + ) + + def _get_prompt_updates( + self, + mm_items: Any, + hf_processor_mm_kwargs: Mapping[str, Any], + out_mm_kwargs: MultiModalKwargsItems, + ) -> Sequence[PromptUpdate]: + out_mm_data = out_mm_kwargs.get_data() + num_patches: Any = out_mm_data.get("num_patches") + num_audio_tokens: Any = out_mm_data.get("num_audio_tokens") + + # Keep the block-start marker and append N placeholder tokens after it; + # only the placeholder positions are flagged as embeddings (is_embed), so + # the marker stays a normal text token while the tower features scatter + # into the placeholders. + def image_replacement(item_idx: int) -> PromptUpdateDetails: + n = int(num_patches[item_idx]) + return PromptUpdateDetails.select_token_id( + [IMAGE_MARKER_ID] + [IMAGE_TOKEN_ID] * n, IMAGE_TOKEN_ID + ) + + def audio_replacement(item_idx: int) -> PromptUpdateDetails: + n = int(num_audio_tokens[item_idx]) + return PromptUpdateDetails.select_token_id( + [AUDIO_MARKER_ID] + [AUDIO_TOKEN_ID] * n, AUDIO_TOKEN_ID + ) + + updates: list[PromptUpdate] = [] + if num_patches is not None and len(num_patches) > 0: + updates.append( + PromptReplacement( + modality="image", + target=[IMAGE_MARKER_ID], + replacement=image_replacement, + ) + ) + if num_audio_tokens is not None and len(num_audio_tokens) > 0: + updates.append( + PromptReplacement( + modality="audio", + target=[AUDIO_MARKER_ID], + replacement=audio_replacement, + ) + ) + return updates diff --git a/vllm/models/inkling/common/towers.py b/vllm/models/inkling/common/towers.py new file mode 100644 index 0000000000..5ac3dbc20e --- /dev/null +++ b/vllm/models/inkling/common/towers.py @@ -0,0 +1,321 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling Titan vision + audio towers. + +The vision tower (``InklingVision`` / ``HMLPPatchEncoder``) emits one token per +image patch; the audio tower (``InklingAudio``) emits one token per audio frame. +Both use vLLM's standard ``RMSNorm`` (CPU-friendly, with a native fallback). +""" + +from __future__ import annotations + +from typing import cast + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from vllm.model_executor.layers.layernorm import RMSNorm + +from ..configs import InklingAudioConfig, InklingVisionConfig + +# =========================================================================== +# Vision tower (HMLPPatchEncoder / InklingVision) +# =========================================================================== + + +def _prime_factors(n: int) -> list[int]: + """Return the prime factors of *n* in ascending order.""" + if n < 1: + raise ValueError("n must be a positive integer") + + factors: list[int] = [] + while n % 2 == 0: + factors.append(2) + n //= 2 + p = 3 + while p * p <= n: + while n % p == 0: + factors.append(p) + n //= p + p += 2 + if n > 1: + factors.append(n) + return factors + + +def plan_out_scales( + temporal_patch_size: int, patch_size: int, n_layers: int, n_channels: int = 3 +) -> list[tuple[int, int, int, int]]: + """Plan the (t, h, w, c) dimensions for each HMLP layer. + + Spatial dims expand first, then temporal; channel counts round up to + multiples of 64. + """ + if patch_size <= 1: + raise ValueError( + "patch_size must be greater than 1, otherwise this doesn't make sense" + ) + + def _round_up(x: int) -> int: + return int(np.ceil(x / 64)) * 64 + + last_h_scale = 1 + scales: list[tuple[int, int, int, int]] = [(1, 1, 1, n_channels)] + for pscale in _prime_factors(patch_size)[::-1]: + last_h_scale *= pscale + scales.append( + ( + 1, + last_h_scale, + last_h_scale, + _round_up((last_h_scale**2) * n_channels), + ) + ) + last_t_scale = 1 + for tscale in _prime_factors(temporal_patch_size)[::-1]: + last_t_scale *= tscale + scales.append( + ( + last_t_scale, + last_h_scale, + last_h_scale, + _round_up((last_h_scale**2) * n_channels * last_t_scale), + ) + ) + + size_reduction = np.prod(np.array(scales)[:, :-1], 1) + + log_ideal_scales = np.linspace( + 0, + np.log(patch_size * patch_size * temporal_patch_size * n_channels), + n_layers + 1, + ) + cost_matrix = np.abs(log_ideal_scales[:, None] - np.log(size_reduction)[None]) + + if n_layers >= len(scales): + idxs = np.argmin(cost_matrix, axis=1) + else: + from scipy.optimize import linear_sum_assignment + + idxs = linear_sum_assignment(cost_matrix)[1] + + assert len(idxs) >= 2 + idxs[0] = 0 + idxs[-1] = len(scales) - 1 + + return [scales[i] for i in idxs] + + +def fold_timespace_to_depth( + vision_patches_bthwc: torch.Tensor, t_fold: int, hw_fold: int +) -> torch.Tensor: + """(B, T, H, W, C) -> (B, T//t, H//hw, W//hw, C*(t*hw**2)).""" + B, T, H, W, C = vision_patches_bthwc.shape + + assert T % t_fold == 0, f"Temporal dimension {T} must be divisible by {t_fold}" + assert H % hw_fold == 0, f"Height dimension {H} must be divisible by {hw_fold}" + assert W % hw_fold == 0, f"Width dimension {W} must be divisible by {hw_fold}" + + t_new = T // t_fold + h_new = H // hw_fold + w_new = W // hw_fold + + x = vision_patches_bthwc.reshape( + B, t_new, t_fold, h_new, hw_fold, w_new, hw_fold, C + ) + x = x.permute(0, 1, 3, 5, 2, 4, 6, 7) + x = x.reshape(B, t_new, h_new, w_new, t_fold * hw_fold * hw_fold * C) + return x + + +class HMLPPatchEncoder(nn.Module): + def __init__(self, config: InklingVisionConfig): + super().__init__() + self.decoder_dmodel = config.decoder_dmodel + self.patch_size = config.patch_size + self.temporal_patch_size = config.temporal_patch_size + self.n_channels = config.n_channels + self.n_layers = config.n_layers + self.use_vision_norm = config.use_vision_norm + + self.scales: list[tuple[int, int, int, int]] = plan_out_scales( + self.temporal_patch_size, self.patch_size, self.n_layers, self.n_channels + ) + self.layers: nn.ModuleDict = nn.ModuleDict() + for i, (start_scale, end_scale) in enumerate( + zip(self.scales[:-1], self.scales[1:]) + ): + shuffle_mult = ( + (end_scale[0] // start_scale[0]) + * (end_scale[1] // start_scale[1]) + * (end_scale[2] // start_scale[2]) + ) + if i == self.n_layers - 1: + self.layers[f"linear_{i}"] = nn.Linear( + start_scale[3] * shuffle_mult, self.decoder_dmodel, bias=False + ) + else: + self.layers[f"linear_{i}"] = nn.Linear( + start_scale[3] * shuffle_mult, end_scale[3], bias=False + ) + self.layers[f"norm_{i}"] = RMSNorm(end_scale[3]) + + self.final_norm: RMSNorm | None = None + if self.use_vision_norm: + assert self.decoder_dmodel is not None + self.final_norm = RMSNorm(self.decoder_dmodel) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # Fused norm+gelu on CUDA (same fp32-accum/bf16-rounding structure as + # the generic path below; differs only by reduction order). + fused = None + if x.is_cuda and x.dtype == torch.bfloat16: + from vllm.models.inkling.nvidia.ops.mm_towers import rmsnorm_gelu + + fused = rmsnorm_gelu + + num_patches, T, H, W, C = x.shape + prefolded = False + for i, (start_scale, end_scale) in enumerate( + zip(self.scales[:-1], self.scales[1:]) + ): + t_fold = end_scale[0] // start_scale[0] + hw_fold = end_scale[1] // start_scale[1] + if (hw_fold > 1 or t_fold > 1) and not prefolded: + x = fold_timespace_to_depth(x, t_fold, hw_fold) + prefolded = False + assert x.shape[1:-1] == ( + T // end_scale[0], + H // end_scale[1], + W // end_scale[2], + ) + x = self.layers[f"linear_{i}"](x) + if i < self.n_layers - 1: + norm = cast(RMSNorm, self.layers[f"norm_{i}"]) + if fused is not None: + # If the NEXT layer starts with a copying fold (spatial + # dims still > 1 after folding), store this layer's + # output directly in the folded layout instead. + nxt = self.scales[i + 2] + ntf = nxt[0] // end_scale[0] + nhf = nxt[1] // end_scale[1] + copy_fold = (ntf > 1 or nhf > 1) and ( + x.shape[2] // nhf > 1 or x.shape[3] // nhf > 1 + ) + x = fused( + x, + norm.weight, + norm.variance_epsilon, + gelu=True, + fold=(ntf, nhf) if copy_fold else None, + ) + prefolded = copy_fold + else: + # rms_norm kernel only supports rank 2-4; x is 5-D here. + orig = x.shape + x = norm(x.reshape(-1, x.shape[-1])).reshape(orig) + x = F.gelu(x) + + if self.final_norm is not None: + if fused is not None: + x = fused( + x, + self.final_norm.weight, + self.final_norm.variance_epsilon, + gelu=False, + ) + else: + orig = x.shape + x = self.final_norm(x.reshape(-1, x.shape[-1])).reshape(orig) + + x = x.reshape(num_patches, -1) + return x + + +class InklingVision(nn.Module): + def __init__(self, config: InklingVisionConfig, prefix: str = ""): + del prefix + super().__init__() + assert config.vision_encoder_type == "hmlp" + self.vision_encoder = HMLPPatchEncoder(config) + + @property + def dtype(self) -> torch.dtype: + return next(self.parameters()).dtype + + @property + def device(self) -> torch.device: + return next(self.parameters()).device + + def forward(self, vision_features: torch.Tensor) -> torch.Tensor: + return self.vision_encoder(vision_features) + + +# =========================================================================== +# Audio tower (InklingAudio) +# =========================================================================== + + +class InklingAudio(nn.Module): + def __init__(self, config: InklingAudioConfig, prefix: str = ""): + del prefix + super().__init__() + assert config.audio_mode == "dmel" + self.n_mel_bins = config.n_mel_bins + self.mel_vocab_size = config.mel_vocab_size + self.use_audio_norm = config.use_audio_norm + self.encoder = nn.Embedding( + config.n_mel_bins * config.mel_vocab_size, config.decoder_dmodel + ) + self.final_norm: RMSNorm | None = None + if self.use_audio_norm: + assert config.decoder_dmodel is not None + self.final_norm = RMSNorm(config.decoder_dmodel, eps=1e-6) + + @property + def dtype(self) -> torch.dtype: + return self.encoder.weight.dtype + + @property + def device(self) -> torch.device: + return self.encoder.weight.device + + def forward(self, audio_features: torch.Tensor) -> torch.Tensor: + assert audio_features.shape[1] == self.n_mel_bins + + # dMel bins are integer indices; cast once to int32 on the right device + # (no float round-trip). + audio_features = audio_features.to( + device=self.encoder.weight.device, dtype=torch.int32 + ) + + weight = self.encoder.weight + if audio_features.is_cuda and weight.dtype == torch.bfloat16: + # One kernel: per-bin offset + embedding gather + fp32 sum + norm. + # Skips the [T, n_mel_bins, D] intermediate entirely (bit-exact). + from vllm.models.inkling.nvidia.ops.mm_towers import dmel_embed_sum_norm + + return dmel_embed_sum_norm( + audio_features.contiguous(), + weight, + self.final_norm.weight if self.final_norm is not None else None, + self.final_norm.variance_epsilon if self.final_norm else 0.0, + ) + + embedding_indices = ( + torch.arange(self.n_mel_bins, device=audio_features.device) + * self.mel_vocab_size + ).unsqueeze(0) + audio_features + + hidden_states = ( + self.encoder(embedding_indices.reshape(-1)) + .reshape(audio_features.shape[0], audio_features.shape[1], -1) + .sum(axis=1) + ) + + if self.final_norm is not None: + hidden_states = self.final_norm(hidden_states) + + return hidden_states diff --git a/vllm/models/inkling/configs.py b/vllm/models/inkling/configs.py new file mode 100644 index 0000000000..f5cbff15cc --- /dev/null +++ b/vllm/models/inkling/configs.py @@ -0,0 +1,373 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling model configs for the text backbone and audio/vision towers.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, ClassVar, Literal, cast + +import torch +from transformers.configuration_utils import PretrainedConfig + + +class InklingModelConfig(PretrainedConfig): + model_type = "inkling_model" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + *, + vocab_size: int = 201024, + hidden_size: int = 1536, + intermediate_size: int = 768, + dense_intermediate_size: int | None = None, + num_hidden_layers: int = 16, + num_attention_heads: int = 12, + num_key_value_heads: int = 4, + head_dim: int | None = None, + v_head_dim: int | None = None, + d_rel: int = 16, + rel_extent: int = 1024, + local_layer_ids: list[int] | None = None, + sliding_window_size: int = 512, + swa_num_attention_heads: int | None = None, + swa_num_key_value_heads: int | None = None, + swa_head_dim: int | None = None, + swa_v_head_dim: int | None = None, + rms_norm_eps: float = 1e-6, + hidden_act: str = "silu", + q_bias: bool = False, + o_bias: bool = False, + use_embed_norm: bool = False, + use_sconv: bool = False, + sconv_kernel_size: int = 4, + dense_mlp_idx: int = 0, + n_routed_experts: int = 0, + n_shared_experts: int = 0, + num_experts_per_tok: int = 1, + route_scale: float = 1.0, + use_gate_bias: bool = False, + use_global_scale: bool = False, + norm_after_topk: bool = True, + gate_activation: Literal["sigmoid", "softmax"] = "sigmoid", + shared_expert_sink: bool = False, + shared_experts_size: int = 1, + inference_moe_w13_interleaved: bool = True, + log_scaling_n_floor: int | None = None, + log_scaling_alpha: float = 0.1, + unpadded_vocab_size: int | None = None, + padded_vocab_size: int | None = None, + logits_mup_width_multiplier: float | None = None, + final_logit_softcapping: float | None = None, + tie_word_embeddings: bool = False, + **kwargs: Any, + ) -> None: + if head_dim is None: + head_dim = hidden_size // num_attention_heads + if v_head_dim is None: + v_head_dim = head_dim + if swa_num_attention_heads is None: + swa_num_attention_heads = num_attention_heads + if swa_num_key_value_heads is None: + swa_num_key_value_heads = num_key_value_heads + if swa_head_dim is None: + swa_head_dim = head_dim + if swa_v_head_dim is None: + swa_v_head_dim = swa_head_dim + if dense_intermediate_size is None: + dense_intermediate_size = intermediate_size + if local_layer_ids is None: + local_layer_ids = [] + + if padded_vocab_size is None: + padded_vocab_size = vocab_size + vocab_size = ( + unpadded_vocab_size + if ( + unpadded_vocab_size is not None + and unpadded_vocab_size < padded_vocab_size + ) + else vocab_size + ) + + self.vocab_size = vocab_size + self.padded_vocab_size = padded_vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.dense_intermediate_size = dense_intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + self.head_dim = head_dim + self.v_head_dim = v_head_dim + self.d_rel = d_rel + self.rel_extent = rel_extent + self.local_layer_ids = local_layer_ids + self.sliding_window_size = sliding_window_size + self.swa_num_attention_heads = swa_num_attention_heads + self.swa_num_key_value_heads = swa_num_key_value_heads + self.swa_head_dim = swa_head_dim + self.swa_v_head_dim = swa_v_head_dim + self.rms_norm_eps = rms_norm_eps + self.hidden_act = hidden_act + self.q_bias = q_bias + self.o_bias = o_bias + self.use_embed_norm = use_embed_norm + self.use_sconv = use_sconv + self.sconv_kernel_size = sconv_kernel_size + self.dense_mlp_idx = dense_mlp_idx + self.n_routed_experts = n_routed_experts + self.num_experts = n_routed_experts + self.n_shared_experts = n_shared_experts + self.num_shared_experts = n_shared_experts + self.num_experts_per_tok = num_experts_per_tok + self.route_scale = route_scale + self.use_gate_bias = use_gate_bias + self.use_global_scale = use_global_scale + self.norm_after_topk = norm_after_topk + self.gate_activation = gate_activation + self.shared_expert_sink = shared_expert_sink + self.shared_experts_size = shared_experts_size + self.inference_moe_w13_interleaved = inference_moe_w13_interleaved + self.log_scaling_n_floor = log_scaling_n_floor + self.log_scaling_alpha = log_scaling_alpha + self.unpadded_vocab_size = self.vocab_size + self.logits_mup_width_multiplier = logits_mup_width_multiplier + self.final_logit_softcapping = final_logit_softcapping + + super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) + + @property + def conv_layer_ids(self) -> list[int]: + return list(range(self.num_hidden_layers)) + + @property + def linear_layer_ids(self) -> list[int]: + return self.conv_layer_ids + + @property + def full_attention_layer_ids(self) -> list[int]: + return list(range(self.num_hidden_layers)) + + @property + def mamba_chunk_size(self) -> int: + # Floor at 64: mamba_cache_chunk_size = max(mamba_chunk_size, page_size), + # and a floor of 1 lets the radix tree adopt another request's KV at + # tiny shared prefixes, whose different kernel-rounding perturbs decode + # logits. + return 64 + + @property + def mamba2_cache_params(self) -> TMLConvCacheParams | None: + try: + from vllm.distributed import get_tensor_model_parallel_world_size + + tp_size = get_tensor_model_parallel_world_size() + except (AssertionError, RuntimeError): + tp_size = 1 + + def tp_local_kv_conv_dim(num_kv_heads: int, head_dim: int) -> int: + return max(1, num_kv_heads // tp_size) * head_dim + + full_kv_conv_dim = tp_local_kv_conv_dim(self.num_key_value_heads, self.head_dim) + local_kv_conv_dim = tp_local_kv_conv_dim( + self.swa_num_key_value_heads, self.swa_head_dim + ) + stream_dim = self.hidden_size + conv_len = self.sconv_kernel_size - 1 + shape = TMLConvStateShape( + conv=[ + (conv_len, full_kv_conv_dim), + (conv_len, full_kv_conv_dim), + (conv_len, local_kv_conv_dim), + (conv_len, local_kv_conv_dim), + (conv_len, stream_dim), + (conv_len, stream_dim), + ], + temporal=(0, 0, 0), + ) + dtype = TMLStateDType(conv=torch.bfloat16, temporal=torch.bfloat16) + return TMLConvCacheParams(shape=shape, layers=self.conv_layer_ids, dtype=dtype) + + +class InklingAudioConfig(PretrainedConfig): + model_type = "inkling_audio_model" + + def __init__( + self, + *, + decoder_dmodel: int | None = None, + n_mel_bins: int | None = None, + mel_vocab_size: int | None = None, + dmel_min_value: float | None = None, + dmel_max_value: float | None = None, + use_audio_norm: bool | None = None, + audio_mode: Literal["dmel", "flow"] | None = None, + **kwargs: Any, + ) -> None: + values = { + "n_mel_bins": n_mel_bins, + "mel_vocab_size": mel_vocab_size, + "dmel_min_value": dmel_min_value, + "dmel_max_value": dmel_max_value, + "use_audio_norm": use_audio_norm, + "audio_mode": audio_mode, + } + if decoder_dmodel is not None and ( + missing := [name for name, value in values.items() if value is None] + ): + raise ValueError( + "Enabled Inkling audio tower is missing config fields: " + + ", ".join(missing) + ) + self.decoder_dmodel = decoder_dmodel + self.n_mel_bins = cast(int, n_mel_bins) + self.mel_vocab_size = cast(int, mel_vocab_size) + self.dmel_min_value = cast(float, dmel_min_value) + self.dmel_max_value = cast(float, dmel_max_value) + self.use_audio_norm = cast(bool, use_audio_norm) + self.audio_mode = cast(Literal["dmel", "flow"], audio_mode) + super().__init__(**kwargs) + + +class InklingVisionConfig(PretrainedConfig): + model_type = "inkling_vision_model" + + def __init__( + self, + *, + vision_encoder_type: Literal["linear", "hmlp"] | None = None, + decoder_dmodel: int | None = None, + patch_size: int | None = None, + temporal_patch_size: int | None = None, + n_channels: int | None = None, + n_layers: int | None = None, + use_vision_norm: bool | None = None, + **kwargs: Any, + ) -> None: + values = { + "vision_encoder_type": vision_encoder_type, + "patch_size": patch_size, + "temporal_patch_size": temporal_patch_size, + "n_channels": n_channels, + "n_layers": n_layers, + "use_vision_norm": use_vision_norm, + } + if decoder_dmodel is not None and ( + missing := [name for name, value in values.items() if value is None] + ): + raise ValueError( + "Enabled Inkling vision tower is missing config fields: " + + ", ".join(missing) + ) + self.vision_encoder_type = cast(Literal["linear", "hmlp"], vision_encoder_type) + self.decoder_dmodel = decoder_dmodel + self.patch_size = cast(int, patch_size) + self.temporal_patch_size = cast(int, temporal_patch_size) + self.n_channels = cast(int, n_channels) + self.n_layers = cast(int, n_layers) + self.use_vision_norm = cast(bool, use_vision_norm) + super().__init__(**kwargs) + + +class InklingMMConfig(PretrainedConfig): + model_type = "inkling_mm_model" + keys_to_ignore_at_inference = ["past_key_values"] + sub_configs: ClassVar[dict[str, type[PretrainedConfig]]] = { + "text_config": InklingModelConfig, + "audio_config": InklingAudioConfig, + "vision_config": InklingVisionConfig, + } + + def __init__( + self, + *, + text_config: dict[str, Any] | InklingModelConfig | None = None, + audio_config: dict[str, Any] | InklingAudioConfig | None = None, + vision_config: dict[str, Any] | InklingVisionConfig | None = None, + tie_word_embeddings: bool = False, + **kwargs: Any, + ) -> None: + self.text_config = ( + text_config + if isinstance(text_config, InklingModelConfig) + else InklingModelConfig(**(text_config or {})) + ) + self.audio_config = ( + audio_config + if isinstance(audio_config, InklingAudioConfig) + else InklingAudioConfig(**(audio_config or {})) + ) + self.vision_config = ( + vision_config + if isinstance(vision_config, InklingVisionConfig) + else InklingVisionConfig(**(vision_config or {})) + ) + super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) + + def get_text_config(self, *args: Any, **kwargs: Any) -> InklingModelConfig: + return self.text_config + + @property + def vocab_size(self) -> int: + return self.text_config.vocab_size + + @property + def hidden_size(self) -> int: + return self.text_config.hidden_size + + @property + def num_hidden_layers(self) -> int: + return self.text_config.num_hidden_layers + + @property + def num_attention_heads(self) -> int: + return self.text_config.num_attention_heads + + @property + def num_key_value_heads(self) -> int: + return self.text_config.num_key_value_heads + + @property + def head_dim(self) -> int: + return self.text_config.head_dim + + @property + def full_attention_layer_ids(self) -> list[int]: + return self.text_config.full_attention_layer_ids + + @property + def linear_layer_ids(self) -> list[int]: + return self.text_config.linear_layer_ids + + @property + def conv_layer_ids(self) -> list[int]: + return self.text_config.conv_layer_ids + + @property + def mamba_chunk_size(self) -> int: + return self.text_config.mamba_chunk_size + + @property + def mamba2_cache_params(self) -> TMLConvCacheParams | None: + return self.text_config.mamba2_cache_params + + +@dataclass(kw_only=True, frozen=True) +class TMLConvStateShape: + conv: list[tuple[int, int]] + temporal: tuple[int, int, int] + + +@dataclass(kw_only=True, frozen=True) +class TMLStateDType: + conv: torch.dtype = torch.bfloat16 + temporal: torch.dtype = torch.bfloat16 + + +@dataclass(kw_only=True, frozen=True) +class TMLConvCacheParams: + shape: TMLConvStateShape + layers: list[int] + dtype: TMLStateDType = field(default_factory=TMLStateDType) diff --git a/vllm/models/inkling/nvfp4.py b/vllm/models/inkling/nvfp4.py new file mode 100644 index 0000000000..129ab82aa1 --- /dev/null +++ b/vllm/models/inkling/nvfp4.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""NVFP4 (ModelOpt) support for the Inkling mixture-of-experts. + +Only the routed MoE experts are quantized in the Inkling checkpoint; +attention, the dense MLP, and the shared "sink" experts stay bf16 (they are +in the checkpoint ``exclude_modules``). The routed experts are served by +vLLM's standard ModelOpt NVFP4 fused-MoE stack (see ``moe.py``); this module +keeps the checkpoint detection. +""" + +from __future__ import annotations + +FLOAT8_E4M3_MAX = 448.0 +FLOAT4_E2M1_MAX = 6.0 + + +class InklingNvfp4Config: + """Lightweight NVFP4 descriptor parsed from the checkpoint quant config. + + Holds the (mapped) ``exclude_modules`` so the model can decide, per MoE + layer and per expert group, whether the weights are NVFP4 or plain bf16. + """ + + def __init__(self, group_size: int, exclude_modules: list[str]) -> None: + self.group_size = group_size + self.exclude_modules = set(exclude_modules) + + @staticmethod + def _is_nvfp4(quant_cfg: dict) -> bool: + wq = quant_cfg["modelopt_quant_config"]["quant_cfg"]["*weight_quantizer"] + return tuple(wq["num_bits"]) == (2, 1) and tuple( + wq["block_sizes"].get("scale_bits", []) + ) == (4, 3) + + @classmethod + def from_hf_config(cls, hf_config) -> InklingNvfp4Config | None: + quant_cfg = getattr(hf_config, "quantization_config", None) + text_config = getattr(hf_config, "text_config", None) + if quant_cfg is None and text_config is not None: + quant_cfg = getattr(text_config, "quantization_config", None) + if quant_cfg is None: + return None + # ModelOpt <=0.29 nests everything under "quantization". + if "quantization" in quant_cfg: + quant_cfg = quant_cfg["quantization"] + if not cls._is_nvfp4(quant_cfg): + return None + group_size = quant_cfg.get("group_size", 16) + if group_size != 16: + raise ValueError("Inkling NVFP4 only supports group size 16") + exclude = list(quant_cfg.get("exclude_modules", []) or []) + return cls(group_size=group_size, exclude_modules=exclude) + + def experts_quantized(self, layer_id: int) -> bool: + """Whether the routed experts of ``layer_id`` are NVFP4 (vs excluded).""" + return f"model.llm.layers.{layer_id}.mlp.experts" not in self.exclude_modules + + def shared_experts_quantized(self, layer_id: int) -> bool: + """Whether the shared sink experts of ``layer_id`` are NVFP4.""" + return ( + f"model.llm.layers.{layer_id}.mlp.shared_experts" + not in self.exclude_modules + ) diff --git a/vllm/models/inkling/nvidia/__init__.py b/vllm/models/inkling/nvidia/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/inkling/nvidia/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/inkling/nvidia/attention.py b/vllm/models/inkling/nvidia/attention.py new file mode 100644 index 0000000000..60440834cf --- /dev/null +++ b/vllm/models/inkling/nvidia/attention.py @@ -0,0 +1,325 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + +from typing import cast + +import torch +from torch import nn + +from vllm.config import VllmConfig, get_current_vllm_config +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.utils.torch_utils import ( + canonicalize_singleton_dim_strides, + kv_cache_dtype_str_to_dtype, +) +from vllm.v1.attention.backend import AttentionBackend +from vllm.v1.attention.backends.flash_attn import ( + FlashAttentionBackend, + FlashAttentionMetadata, +) +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheSpec, + SlidingWindowSpec, +) + +from ..configs import InklingModelConfig +from .layernorm import InklingRMSNorm +from .ops.fa4_rel_attention import ( + bucket_max_seqlen_q, + inkling_fa4_num_splits, + inkling_fa4_rel_attention, +) +from .ops.fa4_warmup import InklingFA4WarmupConfig, register_fa4_warmup +from .ops.qkvr_prep import fused_qkvr_prep +from .sconv_swa_attn import _K, _V, InklingConvState, InklingSconvMetadata +from .short_conv import InklingShortConv + + +def compute_log_scaling_tau( + positions: torch.Tensor, n_floor: int, alpha: float +) -> torch.Tensor: + effective_n = (positions + 1).to(torch.float32) + return 1.0 + alpha * torch.log(torch.clamp(effective_n / float(n_floor), min=1.0)) + + +class RelLogitsProj(nn.Module): + """Project the per-head relative branch ``r`` to per-distance logits.""" + + def __init__(self, d_rel: int, rel_extent: int) -> None: + super().__init__() + self.d_rel = d_rel + self.rel_extent = rel_extent + self.proj = nn.Parameter(torch.empty(d_rel, rel_extent), requires_grad=False) + + def forward(self, r_out: torch.Tensor) -> torch.Tensor: + # r_out: (T, num_heads, d_rel) -> (T, num_heads, rel_extent) + return torch.einsum("thd,de->the", r_out, self.proj) + + +class InklingAttention(nn.Module, AttentionLayerBase): + def __init__( + self, + config: InklingModelConfig, + *, + num_heads: int, + num_kv_heads: int, + head_dim: int, + rel_extent: int, + local_extent: int, + is_local: bool, + prefix: str, + quant_config: QuantizationConfig | None = None, + conv_owner: InklingConvState, + ) -> None: + super().__init__() + self.prefix = prefix + self.is_local = is_local + self.hidden_size = config.hidden_size + self.head_dim = head_dim + self.d_rel = config.d_rel + self.log_scaling_n_floor = config.log_scaling_n_floor + self.log_scaling_alpha = config.log_scaling_alpha + # q/k are per-head RMS-normed (unit norm), so Inkling scales by 1/head_dim. + self.scaling = 1.0 / head_dim + + tp_size = get_tensor_model_parallel_world_size() + self.num_total_heads = num_heads + self.num_total_kv_heads = num_kv_heads + assert self.num_total_heads % tp_size == 0 + self.num_heads = self.num_total_heads // tp_size + if self.num_total_kv_heads >= tp_size: + assert self.num_total_kv_heads % tp_size == 0 + else: + assert tp_size % self.num_total_kv_heads == 0 + self.num_kv_heads = max(1, self.num_total_kv_heads // tp_size) + # When tp_size > num_kv_heads the K/V projections are padded up to + # tp_size heads so each rank gets at least one (GQA replication). + kv_total_for_sizing = max(self.num_total_kv_heads, tp_size) + + self.qkvr = MergedColumnParallelLinear( + input_size=config.hidden_size, + output_sizes=[ + head_dim * self.num_total_heads, + head_dim * kv_total_for_sizing, + head_dim * kv_total_for_sizing, + self.d_rel * self.num_total_heads, + ], + bias=config.q_bias, + quant_config=quant_config, + prefix=f"{prefix}.qkvr", + ) + self.wo_ud = RowParallelLinear( + input_size=head_dim * self.num_total_heads, + output_size=config.hidden_size, + bias=config.o_bias, + quant_config=quant_config, + # reduce_results=False: the partial output is all-reduced below + # (one-shot custom AR) so the attention-output sconv can run on the + # full hidden width fused with the residual add + rmsnorm. + reduce_results=False, + prefix=f"{prefix}.wo_ud", + ) + self.rel_extent = local_extent if is_local else rel_extent + self.local_extent = local_extent if is_local else None + self.rel_logits_proj = RelLogitsProj(self.d_rel, self.rel_extent) + self.q_norm = InklingRMSNorm(head_dim, eps=config.rms_norm_eps) + self.k_norm = InklingRMSNorm(head_dim, eps=config.rms_norm_eps) + + # Short convolution on the K/V streams (per-head-width, TP sharded), + # applied after the qkvr projection and before q/k norm. + kv_conv_dim = self.num_kv_heads * head_dim + self.conv_owner = conv_owner + self.k_sconv = InklingShortConv( + kv_conv_dim, config.sconv_kernel_size, owner=conv_owner, stream_idx=_K + ) + self.v_sconv = InklingShortConv( + kv_conv_dim, config.sconv_kernel_size, owner=conv_owner, stream_idx=_V + ) + + # FA4 left/right window; right=0 keeps it causal. local_extent-1 mirrors + # the source (sliding_window_size - 1). + self.window_size: tuple[int, int] = ( + (local_extent - 1, 0) if is_local else (-1, -1) + ) + # Static per-layer-type KV length bound for the split heuristic: local + # layers never see more than the sliding window. + vllm_config = get_current_vllm_config() + self._max_kv_len = ( + local_extent if is_local else vllm_config.model_config.max_model_len + ) + + # ---- KV-cache wiring (reuse FlashAttentionBackend for metadata) ---- + cache_config = vllm_config.cache_config + self.kv_cache_dtype = ( + cache_config.cache_dtype if cache_config is not None else "auto" + ) + self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype( + self.kv_cache_dtype, vllm_config.model_config + ) + self.register_buffer("k_scale", torch.ones((), dtype=torch.float32)) + self.register_buffer("v_scale", torch.ones((), dtype=torch.float32)) + + compilation_config = vllm_config.compilation_config + if prefix in compilation_config.static_forward_context: + raise ValueError(f"Duplicate layer name: {prefix}") + compilation_config.static_forward_context[prefix] = self + self.kv_cache = torch.tensor([]) # replaced by bind_kv_cache + + register_fa4_warmup( + InklingFA4WarmupConfig( + num_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + head_dim=self.head_dim, + rel_extent=self.rel_extent, + window_size=self.window_size, + is_local=self.is_local, + max_kv_len=self._max_kv_len, + dtype=vllm_config.model_config.dtype, + kv_dtype=self.kv_cache_torch_dtype, + block_size=vllm_config.cache_config.block_size, + max_num_reqs=vllm_config.scheduler_config.max_num_seqs, + max_num_batched_tokens=( + vllm_config.scheduler_config.max_num_batched_tokens + ), + ) + ) + + def get_attn_backend(self) -> type[AttentionBackend]: + return FlashAttentionBackend + + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: + block_size = vllm_config.cache_config.block_size + if self.is_local: + assert self.local_extent is not None + return SlidingWindowSpec( + block_size=block_size, + num_kv_heads=self.num_kv_heads, + head_size=self.head_dim, + dtype=self.kv_cache_torch_dtype, + sliding_window=self.local_extent, + ) + return FullAttentionSpec( + block_size=block_size, + num_kv_heads=self.num_kv_heads, + head_size=self.head_dim, + dtype=self.kv_cache_torch_dtype, + ) + + def _split_kv_cache(self) -> tuple[torch.Tensor, torch.Tensor]: + key_cache, value_cache = self.kv_cache.transpose(1, 2).split( + self.head_dim, dim=-1 + ) + return ( + canonicalize_singleton_dim_strides(key_cache), + canonicalize_singleton_dim_strides(value_cache), + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + log_scaling: torch.Tensor | None = None, + ) -> torch.Tensor: + num_tokens = hidden_states.shape[0] + qkvr, _ = self.qkvr(hidden_states) + + attn_metadata = get_forward_context().attn_metadata + attn_output = torch.empty( + (num_tokens, self.num_heads, self.head_dim), + dtype=qkvr.dtype, + device=qkvr.device, + ) + if not isinstance(attn_metadata, dict): + attn_output.zero_() + else: + conv_meta = attn_metadata[self.conv_owner.prefix] + md = attn_metadata[self.prefix] + assert isinstance(conv_meta, InklingSconvMetadata) + fa_md = cast(FlashAttentionMetadata, md) + assert self.kv_cache.numel() > 0 + assert self.conv_owner.kv_cache.numel() > 0 + # One launch: K/V sconv (conv-cache insert + conv + residual), + # Q/K per-head rmsnorm, and the attention KV-cache write. K/V are + # consumed via the KV cache; only normed q is materialized. + key_cache, value_cache = self._split_kv_cache() + off_k, _ = self.conv_owner.stream_ranges[_K] + off_v, _ = self.conv_owner.stream_ranges[_V] + q, rel_logits = fused_qkvr_prep( + qkvr, + self.k_sconv.weight.squeeze(1), + self.v_sconv.weight.squeeze(1), + self.q_norm.weight, + self.k_norm.weight, + self.rel_logits_proj.proj, + self.q_norm.variance_epsilon, + self.num_heads, + self.num_kv_heads, + self.head_dim, + self.d_rel, + self.conv_owner.kv_cache, + key_cache, + value_cache, + positions, + conv_meta.block_table, + conv_meta.seq_idx, + conv_meta.slot_mapping, + conv_meta.query_start, + fa_md.slot_mapping, + off_k, + off_v, + self.conv_owner.block_size, + log_scaling if not self.is_local else None, + ) + q = q.view(num_tokens, self.num_heads, self.head_dim) + self._attention(q, rel_logits, attn_output) + + flat = attn_output.view(num_tokens, -1) + output, _ = self.wo_ud(flat) + return output + + def _attention( + self, + q: torch.Tensor, + rel_logits: torch.Tensor, + output: torch.Tensor, + ) -> None: + attn_metadata = get_forward_context().attn_metadata + assert isinstance(attn_metadata, dict) + md = cast(FlashAttentionMetadata, attn_metadata[self.prefix]) + + nt = md.num_actual_tokens + key_cache, value_cache = self._split_kv_cache() + max_seqlen_q = bucket_max_seqlen_q(md.max_query_len) + num_splits = inkling_fa4_num_splits( + is_local=self.is_local, + batch_size=md.seq_lens.shape[0], + max_query_len=max_seqlen_q, + num_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + max_kv_len=self._max_kv_len, + ) + inkling_fa4_rel_attention( + q[:nt], + key_cache, + value_cache, + block_table=md.block_table, + cache_seqlens=md.seq_lens, + cu_seqlens_q=md.query_start_loc, + max_seqlen_q=max_seqlen_q, + softmax_scale=self.scaling, + causal=True, + window_size=self.window_size, + rel_extent=self.rel_extent, + rel_logits=rel_logits[:nt], + num_splits=num_splits, + out=output[:nt], + ) diff --git a/vllm/models/inkling/nvidia/layernorm.py b/vllm/models/inkling/nvidia/layernorm.py new file mode 100644 index 0000000000..18f9951f65 --- /dev/null +++ b/vllm/models/inkling/nvidia/layernorm.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling RMSNorm (no bias, weight-scaled), backed by the vendored Triton kernel.""" + +from __future__ import annotations + +import torch +from torch import nn + +from .ops import rmsnorm + + +class InklingRMSNorm(nn.Module): + def __init__(self, hidden_size: int, eps: float = 1e-6) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + self.hidden_size = hidden_size + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if x.numel() == 0: + return x + original_shape = x.shape + x_2d = x.contiguous().view(-1, self.hidden_size) + y = rmsnorm(x_2d, self.weight, self.variance_epsilon) + return y.view(original_shape) diff --git a/vllm/models/inkling/nvidia/logits_processor.py b/vllm/models/inkling/nvidia/logits_processor.py new file mode 100644 index 0000000000..b1b82659ff --- /dev/null +++ b/vllm/models/inkling/nvidia/logits_processor.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling logits processor with muP output scaling.""" + +from __future__ import annotations + +import torch + +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding + + +class InklingLogitsProcessor(LogitsProcessor): + """``LogitsProcessor`` that applies Inkling's muP logits width multiplier. + + Args: + vocab_size: Padded vocabulary size. + org_vocab_size: Unpadded vocabulary size (defaults to ``vocab_size``). + scale: Base logits scale (kept ``1.0`` for the served checkpoint). + logits_as_input: Whether the input is already logits. + soft_cap: Optional logit soft cap (``None`` for the served checkpoint). + logits_mup_width_multiplier: muP width divisor for the final logits; + ``None`` or ``0`` disables it. + """ + + def __init__( + self, + vocab_size: int, + org_vocab_size: int | None = None, + scale: float = 1.0, + logits_as_input: bool = False, + soft_cap: float | None = None, + logits_mup_width_multiplier: float | None = None, + ) -> None: + super().__init__( + vocab_size=vocab_size, + org_vocab_size=org_vocab_size, + scale=scale, + logits_as_input=logits_as_input, + soft_cap=soft_cap, + ) + self.logits_mup_width_multiplier = logits_mup_width_multiplier + self._logits_zero: torch.Tensor | None = None + + def forward( + self, + lm_head: VocabParallelEmbedding, + hidden_states: torch.Tensor, + embedding_bias: torch.Tensor | None = None, + ) -> torch.Tensor | None: + mup = self.logits_mup_width_multiplier + if not mup: + return super().forward(lm_head, hidden_states, embedding_bias) + # Fold the muP width divisor into the lm_head GEMM alpha (fp32 epilogue): + # no separate elementwise kernel, no bf16 rounding of scaled logits, and + # no weight mutation. Overfit to the served checkpoint: bf16 lm_head, no + # soft cap, unit logits scale. + assert self.soft_cap is None + assert self.scale == 1.0 + w = lm_head.weight + if self._logits_zero is None: + self._logits_zero = w.new_zeros(1) + logits = torch.addmm( + self._logits_zero, + hidden_states, + w.t(), + beta=0.0, + alpha=1.0 / mup, + ) + logits = self._gather_logits(logits) + if logits is not None: + logits = logits[..., : self.org_vocab_size] + return logits diff --git a/vllm/models/inkling/nvidia/mlp.py b/vllm/models/inkling/nvidia/mlp.py new file mode 100644 index 0000000000..ab7dec0b70 --- /dev/null +++ b/vllm/models/inkling/nvidia/mlp.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling dense SwiGLU MLP (also used as the MoE shared expert). + +The checkpoint stores the gate/up projection as a single fused, *interleaved* +weight (``[gate0, up0, gate1, up1, ...]``), so we use a plain +``ColumnParallelLinear`` whose contiguous row-sharding keeps each gate/up pair +together, and an interleaved SwiGLU activation. +""" + +from __future__ import annotations + +import torch +from torch import nn + +from vllm.model_executor.layers.linear import ( + ColumnParallelLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.quantization import QuantizationConfig + + +class InklingDenseMLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + *, + use_global_scale: bool = False, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = ColumnParallelLinear( + hidden_size, + 2 * intermediate_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.down_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=False, + prefix=f"{prefix}.down_proj", + ) + if use_global_scale: + self.global_scale = nn.Parameter(torch.empty(1), requires_grad=False) + else: + self.global_scale = None + + def forward(self, x: torch.Tensor) -> torch.Tensor: + from .ops import silu_and_mul_triton + + gate_up, _ = self.gate_up_proj(x) + x = silu_and_mul_triton(gate_up) + x, _ = self.down_proj(x) + if self.global_scale is not None: + x = x * self.global_scale + # TP-partial output: the layer's reduce-scatter fallback consumes it. + return x diff --git a/vllm/models/inkling/nvidia/model.py b/vllm/models/inkling/nvidia/model.py new file mode 100644 index 0000000000..0f62af3a4f --- /dev/null +++ b/vllm/models/inkling/nvidia/model.py @@ -0,0 +1,678 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling model implementation for NVIDIA GPUs.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +import regex as re +import torch +from torch import nn + +from vllm.config import VllmConfig +from vllm.distributed import ( + get_pp_group, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_gather, + tensor_model_parallel_reduce_scatter, +) +from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.models.interfaces import ( + MultiModalEmbeddings, + SupportsMultiModal, + SupportsPP, +) +from vllm.model_executor.models.utils import ( + AutoWeightsLoader, + WeightsMapper, + make_empty_intermediate_tensors_factory, + make_layers, + maybe_prefix, +) +from vllm.models.inkling.common.mm_preprocess import ( + InklingDummyInputsBuilder, + InklingMultiModalProcessor, + InklingProcessingInfo, + inkling_audio_enabled, + inkling_vision_enabled, +) +from vllm.models.inkling.common.towers import InklingAudio, InklingVision +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.sequence import IntermediateTensors + +from ..configs import InklingMMConfig, InklingModelConfig +from ..nvfp4 import InklingNvfp4Config +from .attention import InklingAttention, compute_log_scaling_tau +from .layernorm import InklingRMSNorm +from .logits_processor import InklingLogitsProcessor +from .mlp import InklingDenseMLP +from .moe import InklingMoE +from .ops.lamport import get_lamport_rs_conv, initialize_lamport_rs_conv +from .ops.norm import add_rmsnorm, embed_rmsnorm +from .sconv_swa_attn import _ATTN, _MLP, InklingConvState, InklingSconvMetadata +from .short_conv import InklingShortConv + + +def _layer_id(name: str) -> int | None: + m = re.search(r"\.layers\.(\d+)\.", name) + return int(m.group(1)) if m else None + + +def _sconv_add_norm( + delta: torch.Tensor, + hidden: torch.Tensor, + sconv: InklingShortConv, + norm: InklingRMSNorm | None, + positions: torch.Tensor, +) -> tuple[torch.Tensor | None, torch.Tensor]: + """``h = hidden + sconv(TP-sum(delta)); y = rmsnorm(h)``. + + The Lamport path performs reduce-scatter + shard sconv + all-gather + + residual add + norm. The NCCL path handles unsupported configurations.""" + attn_metadata = get_forward_context().attn_metadata + m = ( + attn_metadata.get(sconv.owner.prefix) + if isinstance(attn_metadata, dict) + else None + ) + cache = sconv.owner.kv_cache + off_s, ws = sconv.owner.stream_ranges[sconv.stream_idx] + norm_w = norm.weight if norm is not None else None + eps = norm.variance_epsilon if norm is not None else 0.0 + + mm = get_lamport_rs_conv(hidden.shape[-1], sconv.kernel_size) + if mm is not None and mm.usable(delta.shape[0]) and m is not None: + assert cache.numel() > 0 + assert isinstance(m, InklingSconvMetadata) + return mm.rs_sconv_ag_add_norm( + delta, + hidden, + sconv.weight.squeeze(1), + norm_w, + eps, + cache, + positions, + m.block_table, + m.seq_idx, + m.slot_mapping, + off_s, + ws, + sconv.owner.block_size, + ) + + # Fallback: NCCL RS -> shard sconv -> AG -> fused add(+rmsnorm). + shard = tensor_model_parallel_reduce_scatter(delta, dim=-1) + shard = sconv(shard.contiguous(), positions) + full = tensor_model_parallel_all_gather(shard, dim=-1) + if norm is None: + return None, hidden + full + return add_rmsnorm(hidden, full, norm_w, eps) + + +class InklingDecoderLayer(nn.Module): + def __init__( + self, + config: InklingModelConfig, + layer_id: int, + is_local: bool, + quant_config: QuantizationConfig | None, + prefix: str, + nvfp4_config: InklingNvfp4Config | None = None, + ) -> None: + super().__init__() + # Per-layer owner of the conv state as a paged SWA cache. The 4 sconv + # streams (K/V/attn/mlp) are packed head-major into one block and share + # it. Built first so the attention layer can wire its K/V sconv to it. + self.conv_state = InklingConvState( + num_kv_heads=( + config.swa_num_key_value_heads + if is_local + else config.num_key_value_heads + ), + head_dim=config.swa_head_dim if is_local else config.head_dim, + hidden_size=config.hidden_size, + kernel_size=config.sconv_kernel_size, + prefix=f"{prefix}.conv_state", + ) + self.attn_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.attn = InklingAttention( + config, + num_heads=( + config.swa_num_attention_heads + if is_local + else config.num_attention_heads + ), + num_kv_heads=( + config.swa_num_key_value_heads + if is_local + else config.num_key_value_heads + ), + head_dim=config.swa_head_dim if is_local else config.head_dim, + rel_extent=config.rel_extent, + local_extent=config.sliding_window_size, + is_local=is_local, + prefix=f"{prefix}.attn", + quant_config=quant_config, + conv_owner=self.conv_state, + ) + self.mlp_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + if layer_id < config.dense_mlp_idx: + self.mlp: nn.Module = InklingDenseMLP( + hidden_size=config.hidden_size, + intermediate_size=config.dense_intermediate_size, + use_global_scale=config.use_global_scale, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + else: + # InklingMoE decides per layer (from the checkpoint exclude list) + # whether the routed experts are NVFP4 or bf16; the shared sink + # experts are always bf16. + self.mlp = InklingMoE( + config, + layer_id, + prefix=f"{prefix}.mlp", + nvfp4_config=nvfp4_config, + ) + + # Short convolution on the attention-output and MLP-output residual + # streams, hidden-sharded: the sublayer outputs are reduce-scattered + # to [T, H/tp], the sconv runs on the shard, and an all-gather + # restores the full residual — all fused with the residual add + next + # rmsnorm via the Lamport P2P kernels for decode-sized batches. + tp_size = get_tensor_model_parallel_world_size() + sconv_dim = config.hidden_size // tp_size + self.attn_sconv = InklingShortConv( + sconv_dim, config.sconv_kernel_size, owner=self.conv_state, stream_idx=_ATTN + ) + self.mlp_sconv = InklingShortConv( + sconv_dim, config.sconv_kernel_size, owner=self.conv_state, stream_idx=_MLP + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + pending: tuple[torch.Tensor | None, InklingShortConv] | None = None, + attn_in: torch.Tensor | None = None, + log_scaling: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, tuple[torch.Tensor | None, InklingShortConv]]: + # The previous sublayer's (pre-reduce, pre-sconv) delta is folded in + # fused with its RS/sconv/AG and this layer's pre-attention rmsnorm. + # A None delta means the partials sit in the NVLS symm buffer. + if pending is None: + if attn_in is None: + # First layer; on the text path attn_norm comes fused with + # the embedding gather (chain_weight in embed_rmsnorm). + attn_in = self.attn_norm(hidden_states) + else: + attn_in, hidden_states = _sconv_add_norm( + pending[0], hidden_states, pending[1], self.attn_norm, positions + ) + attn_output = self.attn(positions, attn_in, log_scaling) + mlp_in, hidden_states = _sconv_add_norm( + attn_output, hidden_states, self.attn_sconv, self.mlp_norm, positions + ) + mlp_output = self.mlp(mlp_in) + # The caller folds mlp_output (pre-reduce, pre-sconv) into the next + # fused sconv+add+rmsnorm. + return hidden_states, (mlp_output, self.mlp_sconv) + + +class InklingReplicatedEmbedding(nn.Module): + """Full-vocab embedding table replicated on every TP rank. + + Trades the full table per rank (~2.3 GiB at V=201k / H=6144 bf16, vs a + 1/tp shard) for no masked lookup or per-lookup TP all-reduce, and keeps the + full table on-rank for the fused gather+norm kernel. Bit-exact vs + vocab-parallel: the all-reduce there only ever summed one real row against + exact zeros. The LM head stays vocab-sharded. + """ + + def __init__(self, num_embeddings: int, embedding_dim: int) -> None: + super().__init__() + self.weight = nn.Parameter( + torch.empty(num_embeddings, embedding_dim, dtype=torch.get_default_dtype()), + requires_grad=False, + ) + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + return embed_rmsnorm(input_ids, self.weight, None, 0.0) + + +class InklingModel(nn.Module): + def __init__( + self, + *, + config: InklingModelConfig, + quant_config: QuantizationConfig | None, + prefix: str, + nvfp4_config: InklingNvfp4Config | None = None, + ) -> None: + super().__init__() + self.config = config + self.embed_tokens = InklingReplicatedEmbedding( + config.padded_vocab_size, config.hidden_size + ) + self.embed_norm = ( + InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + if config.use_embed_norm + else None + ) + local_ids = set(config.local_layer_ids) + + def get_layer(prefix: str) -> InklingDecoderLayer: + idx = _layer_id(prefix + ".") or int(prefix.split(".")[-1]) + return InklingDecoderLayer( + config, idx, idx in local_ids, quant_config, prefix, nvfp4_config + ) + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers" + ) + self.norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( + ["hidden_states"], config.hidden_size + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + # Row gather + embed_norm in one launch. + norm = self.embed_norm + return embed_rmsnorm( + input_ids, + self.embed_tokens.weight, + norm.weight if norm is not None else None, + norm.variance_epsilon if norm is not None else 0.0, + ) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None, + inputs_embeds: torch.Tensor | None = None, + ) -> torch.Tensor | IntermediateTensors: + attn_in0: torch.Tensor | None = None + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + # embed_norm was already applied when producing inputs_embeds. + hidden_states = inputs_embeds + else: + # Gather + embed_norm + the first layer's attn_norm, one launch. + norm = self.embed_norm + hidden_states, attn_in0 = embed_rmsnorm( + input_ids, + self.embed_tokens.weight, + norm.weight if norm is not None else None, + self.config.rms_norm_eps, + chain_weight=self.layers[self.start_layer].attn_norm.weight, + ) + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + hidden_states = hidden_states.view(-1, hidden_states.shape[-1]) + log_scaling = None + if self.config.log_scaling_n_floor is not None: + log_scaling = compute_log_scaling_tau( + positions, + self.config.log_scaling_n_floor, + self.config.log_scaling_alpha, + ) + + pending: tuple[torch.Tensor | None, InklingShortConv] | None = None + for layer in self.layers[self.start_layer : self.end_layer]: + hidden_states, pending = layer( + positions, + hidden_states, + pending=pending, + attn_in=attn_in0, + log_scaling=log_scaling, + ) + attn_in0 = None + + if not get_pp_group().is_last_rank: + if pending is not None: + hidden_states = _sconv_add_norm( + pending[0], hidden_states, pending[1], None, positions + )[1] + return IntermediateTensors({"hidden_states": hidden_states}) + if pending is not None: + # Final RS/sconv/AG + residual add fused with the final rmsnorm. + norm_out = _sconv_add_norm( + pending[0], hidden_states, pending[1], self.norm, positions + )[0] + assert norm_out is not None + return norm_out + return self.norm(hidden_states) + + +class _TmlForCausalLMBase(nn.Module, SupportsPP): + """Shared text-backbone causal-LM scaffolding for both entry classes.""" + + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={ + ".w13_dn": ".gate_up_proj", + ".w2_md": ".down_proj", + }, + orig_to_new_stacked={ + ".attn.wq_du.": (".attn.qkvr.", 0), + ".attn.wk_dv.": (".attn.qkvr.", 1), + ".attn.wv_dv.": (".attn.qkvr.", 2), + ".attn.wr_du.": (".attn.qkvr.", 3), + }, + orig_to_new_prefix={ + "model.llm.layers.": "model.layers.", + "model.llm.embed_norm": "model.embed_norm", + "model.llm.embed": "model.embed_tokens", + "model.llm.norm": "model.norm", + "model.llm.unembed": "lm_head", + }, + orig_to_new_suffix={ + # NVFP4 scale + ".w13_weight.scale": ".w13_weight_scale", + ".w13_weight.scale2": ".w13_weight_scale_2", + ".w2_weight.scale": ".w2_weight_scale", + ".w2_weight.scale2": ".w2_weight_scale_2", + }, + ) + + def _build( + self, + vllm_config: VllmConfig, + text_config: InklingModelConfig, + prefix: str, + ) -> None: + quant_config = vllm_config.quant_config + self.config = text_config + # NVFP4 experts are detected directly from the checkpoint quant config; + # only the MoE experts are quantized (attention/dense MLP stay bf16). + self.nvfp4_config = InklingNvfp4Config.from_hf_config( + vllm_config.model_config.hf_config + ) + # Read by the MRV2 runner to publish per-request short-conv metadata. + # Short convolution is intrinsic to Inkling, so this is always set. + self.uses_sconv = True + self.model = InklingModel( + config=text_config, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "model"), + nvfp4_config=self.nvfp4_config, + ) + initialize_lamport_rs_conv( + text_config.hidden_size, + text_config.sconv_kernel_size, + vllm_config.scheduler_config.max_num_batched_tokens, + ) + self.lm_head = ParallelLMHead( + text_config.padded_vocab_size, + text_config.hidden_size, + org_num_embeddings=text_config.padded_vocab_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + self.logits_processor = InklingLogitsProcessor( + text_config.padded_vocab_size, + org_vocab_size=text_config.vocab_size, + soft_cap=text_config.final_logit_softcapping, + logits_mup_width_multiplier=text_config.logits_mup_width_multiplier, + ) + self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] + self.model.make_empty_intermediate_tensors + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs: object, + ) -> torch.Tensor | IntermediateTensors: + return self.model( + input_ids, + positions, + intermediate_tensors, + inputs_embeds, + ) + + def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: + return self.logits_processor(self.lm_head, hidden_states) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + return _load_inkling_weights(self, weights, self.config) + + +class InklingForCausalLM(_TmlForCausalLMBase): + """Text-only entry point (``inkling_model`` checkpoints).""" + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + self._build(vllm_config, vllm_config.model_config.hf_config, prefix) + + +@MULTIMODAL_REGISTRY.register_processor( + InklingMultiModalProcessor, + info=InklingProcessingInfo, + dummy_inputs=InklingDummyInputsBuilder, +) +class InklingForConditionalGeneration(_TmlForCausalLMBase, SupportsMultiModal): + """Top-level (multimodal) entry point. + + Builds the vision + audio towers on top of the shared text backbone. Inkling has + NO cross-modal fusion (the vision tower emits one token per patch, the audio + tower one token per frame), so generation reuses the inherited backbone + ``forward`` / ``compute_logits`` (the latter already applies muP) and this + class only adds multimodal embedding + merge. + """ + + hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_prefix={ + "model.audio.": "audio.", + "model.visual.": "visual.vision_encoder.", + }, + ) + + @classmethod + def get_placeholder_str(cls, modality: str, i: int) -> str | None: + if modality.startswith("image"): + return "<|content_image|>" + if modality.startswith("audio"): + return "<|content_audio_input|>" + raise ValueError("Only image or audio modality is supported") + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: + super().__init__() + config: InklingMMConfig = vllm_config.model_config.hf_config + + self.visual = ( + InklingVision(config.vision_config, prefix=maybe_prefix(prefix, "visual")) + if inkling_vision_enabled(config) + else None + ) + self.audio = ( + InklingAudio(config.audio_config, prefix=maybe_prefix(prefix, "audio")) + if inkling_audio_enabled(config) + else None + ) + + self._build(vllm_config, config.text_config, prefix) + + # -- multimodal embedding ------------------------------------------- + + def _process_image_input( + self, pixel_values: Any, num_patches: Any + ) -> tuple[torch.Tensor, ...]: + assert self.visual is not None + # pixel_values is a list (per item) of [P_i, 2, P, P, 3] tensors, + # or a single concatenated tensor. Normalize to a flat batch, run the + # tower once, then split back per item. + if isinstance(pixel_values, (list, tuple)): + if not pixel_values: + return () + sizes = [int(p.shape[0]) for p in pixel_values] + patches = torch.cat(list(pixel_values), dim=0) + else: + patches = pixel_values + sizes = self._sizes_from(num_patches, patches.shape[0]) + + patches = patches.to(device=self.visual.device, dtype=self.visual.dtype) + embeds = self.visual(patches) # [total_patches, D] + return tuple(embeds.split(sizes)) + + def _process_audio_input( + self, input_audio_features: Any, num_audio_tokens: Any + ) -> tuple[torch.Tensor, ...]: + assert self.audio is not None + if isinstance(input_audio_features, (list, tuple)): + if not input_audio_features: + return () + sizes = [int(d.shape[0]) for d in input_audio_features] + dmel = torch.cat(list(input_audio_features), dim=0) + else: + dmel = input_audio_features + sizes = self._sizes_from(num_audio_tokens, dmel.shape[0]) + + dmel = dmel.to(device=self.audio.device) + embeds = self.audio(dmel) # [total_frames, D] + return tuple(embeds.split(sizes)) + + @staticmethod + def _sizes_from(counts: Any, total: int) -> list[int]: + if counts is None: + return [total] + if isinstance(counts, torch.Tensor): + return [int(c) for c in counts.flatten().tolist()] + if isinstance(counts, (list, tuple)): + flat: list[int] = [] + for c in counts: + flat.append(int(c.item()) if isinstance(c, torch.Tensor) else int(c)) + return flat + return [int(counts)] + + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: + # Iterate modalities in a stable order so the returned per-item tensors + # line up with their appearance order; the positional merge in + # embed_input_ids handles actual placement. + pixel_values = kwargs.get("pixel_values") + num_patches = kwargs.get("num_patches") + input_audio_features = kwargs.get("input_audio_features") + num_audio_tokens = kwargs.get("num_audio_tokens") + + embeddings: tuple[torch.Tensor, ...] = () + if pixel_values is not None and self.visual is not None: + embeddings += self._process_image_input(pixel_values, num_patches) + if input_audio_features is not None and self.audio is not None: + embeddings += self._process_audio_input( + input_audio_features, num_audio_tokens + ) + return embeddings + + def embed_input_ids( + self, + input_ids: torch.Tensor, + multimodal_embeddings: MultiModalEmbeddings | None = None, + *, + is_multimodal: torch.Tensor | None = None, + ) -> torch.Tensor: + # Override the base's 1-arg embed_input_ids: the runner calls this 3-arg + # signature for multimodal models. Text embeddings come from the shared + # backbone (which applies embed_norm); MM embeddings are scattered in. + from vllm.model_executor.models.utils import _merge_multimodal_embeddings + + # Placeholder ids use unused vocabulary slots and these positions are + # overwritten by MM embeds below. + inputs_embeds = self.model.embed_input_ids(input_ids) + if multimodal_embeddings is None or len(multimodal_embeddings) == 0: + return inputs_embeds + assert is_multimodal is not None + return _merge_multimodal_embeddings( + inputs_embeds=inputs_embeds, + multimodal_embeddings=multimodal_embeddings, + is_multimodal=is_multimodal, + ) + + def get_language_model(self) -> nn.Module: + # This class IS the causal LM (the towers are side branches), so the + # language model is self — callers expect a module exposing ``.model`` + # and ``.lm_head``. + return self + + +# =========================================================================== +# Weight loading +# =========================================================================== + + +_MOE_EXPERT_WEIGHT_RE = re.compile( + r"^(?P.*\.mlp)\.(?P(?:shared_)?experts\..+)$" +) + + +def _load_inkling_weights( + module: nn.Module, + weights: Iterable[tuple[str, torch.Tensor]], + config: InklingModelConfig, +) -> set[str]: + moe_modules = { + name: mod for name, mod in module.named_modules() if isinstance(mod, InklingMoE) + } + loaded: set[str] = set() + tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() + local_ids = set(config.local_layer_ids) + + def _iter_loadable_weights() -> Iterable[tuple[str, torch.Tensor]]: + for name, weight in module.hf_to_vllm_mapper.apply(weights): + shard_id = getattr(weight, "shard_id", None) + # Replicate K/V conv-free GQA heads when tp_size > num_kv_heads. + if ( + shard_id in (1, 2) + and name.endswith(".attn.qkvr.weight") + and weight.shape[0] > 0 + ): + lid = _layer_id(name) + if lid is not None: + is_local = lid in local_ids + n_kv = ( + config.swa_num_key_value_heads + if is_local + else config.num_key_value_heads + ) + head_dim = config.swa_head_dim if is_local else config.head_dim + if tp_size > n_kv and weight.shape[0] == n_kv * head_dim: + kv_idx = (tp_rank * n_kv) // tp_size + weight = weight.narrow(0, kv_idx * head_dim, head_dim) + weight.shard_id = shard_id + + # MoE expert tensors (fused stacked, routed + shared sink): translate + # the checkpoint layout to per-expert FusedMoE loads. + moe_match = _MOE_EXPERT_WEIGHT_RE.match(name) + if moe_match is not None and moe_match.group("mlp") in moe_modules: + moe = moe_modules[moe_match.group("mlp")] + for rel in moe.load_expert_weight(moe_match.group("rest"), weight): + loaded.add(f"{moe_match.group('mlp')}.{rel}") + continue + + yield name, weight + + # The release checkpoint also carries auxiliary prediction-head weights; + # they are not part of the causal LM served by this implementation. + loader = AutoWeightsLoader(module, skip_prefixes=["model.mtp."]) + loaded |= loader.load_weights(_iter_loadable_weights()) + + # Post-load MoE fixups (default input scales, zeroed EP-padding experts). + for moe_name, moe in moe_modules.items(): + for rel in moe.finalize_load(): + loaded.add(f"{moe_name}.{rel}") + return loaded + + +EntryClass = [InklingForCausalLM, InklingForConditionalGeneration] diff --git a/vllm/models/inkling/nvidia/moe.py b/vllm/models/inkling/nvidia/moe.py new file mode 100644 index 0000000000..3bb281c963 --- /dev/null +++ b/vllm/models/inkling/nvidia/moe.py @@ -0,0 +1,578 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling mixture-of-experts on vLLM's FusedMoE abstraction. + +Overfit to the served checkpoint: sigmoid gate (+ selection bias) top-k over +the routed experts, log-sigmoid renormalization over the k routed + S shared +"sink" logits, scaled by route_scale * global_scale. The routed top-k goes +through vLLM's FusedMoE (which handles TP/EP); the sink experts run in +:class:`InklingSinkExperts` -- replicated across EP ranks (every token +activates every sink) and always bf16 (the checkpoint excludes every +``shared_experts`` from quantization). + +NVFP4 routed experts reuse vLLM's ModelOpt NVFP4 fused-MoE method; excluded +(bf16) layers fall back to the unquantized method. The checkpoint's fused +stacked tensors (interleaved gate/up rows, ``.scale`` / ``.scale2`` / +``.input_amax`` aux tensors) are translated to the standard per-expert loads +in :meth:`InklingMoE.load_expert_weight`. +""" + +from __future__ import annotations + +import math +from typing import TYPE_CHECKING + +import torch +from torch import nn +from torch.nn.parameter import Parameter + +import vllm.envs as envs +from vllm.config import get_current_vllm_config +from vllm.distributed import ( + get_dp_group, + get_pcp_group, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) +from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16 +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.utils import set_weight_attrs +from vllm.platforms import current_platform +from vllm.triton_utils import tl, tldevice, triton +from vllm.utils.multi_stream_utils import maybe_execute_in_parallel +from vllm.utils.torch_utils import aux_stream + +from ..configs import InklingModelConfig +from ..nvfp4 import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX + +if TYPE_CHECKING: + from vllm.model_executor.layers.fused_moe.routed_experts import ( + RoutedExperts, + ) + + from ..nvfp4 import InklingNvfp4Config + +# --------------------------------------------------------------------------- +# Gate / expert selection +# --------------------------------------------------------------------------- + +_INKLING_LL_BF16_MAX_TOKENS = 64 + + +def _linear_with_fp32_out(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: + leading = list(x.shape[:-1]) + flat = x.flatten(0, -2) + if ( + flat.shape[0] <= _INKLING_LL_BF16_MAX_TOKENS + and flat.dtype == torch.bfloat16 + and weight.dtype == torch.bfloat16 + and flat.is_cuda + and flat.is_contiguous() + and weight.is_contiguous() + and flat.shape[1] % 8 == 0 + and current_platform.has_device_capability(90) + and ll_bf16.is_available() + ): + out = ll_bf16.ll_bf16_gemm(flat, weight) + else: + out = torch.mm(flat, weight.T, out_dtype=torch.float32) + return out.view(*leading, weight.shape[0]) + + +@triton.jit(do_not_specialize=["T", "route_scale"]) +def _inkling_gate_select_kernel( + logits_ptr, # [T, G] fp32 gate logits (stride_logits_0 may include pad) + bias_ptr, # [R] fp32 selection bias (or 0 ptr if HAS_BIAS=False) + global_scale_ptr, # [1] fp32 (or unused if HAS_GSCALE=False) + ids_ptr, # [T, K + S] int32 out: selected expert ids + weights_ptr, # [T, K + S] fp32 out: renormalized weights + route_scale, + T, + G: tl.constexpr, # total gate experts (routed + shared) + stride_logits_0, + R: tl.constexpr, # routed experts + K: tl.constexpr, # top-k routed + S: tl.constexpr, # shared (sink) experts + HAS_BIAS: tl.constexpr, + HAS_GSCALE: tl.constexpr, + BLOCK_G: tl.constexpr, +): + pid = tl.program_id(0).to(tl.int64) + if pid >= T: + return + offs = tl.arange(0, BLOCK_G) + mask_r = offs < R + logits = tl.load( + logits_ptr + pid * stride_logits_0 + offs, + mask=offs < G, + other=float("-inf"), + ).to(tl.float32) + + # Selection scores: sigmoid(routed logits) (+ bias), non-routed lanes -inf. + sel = tl.where(mask_r, tl.sigmoid(logits), float("-inf")) + if HAS_BIAS: + bias = tl.load(bias_ptr + offs, mask=mask_r, other=0.0).to(tl.float32) + sel = tl.where(mask_r, sel + bias, float("-inf")) + + scale = route_scale + if HAS_GSCALE: + scale = scale * tl.load(global_scale_ptr).to(tl.float32) + + # Iterative top-K (K is small); argmax tie-breaks to the lowest index + # (stable ordering). + A: tl.constexpr = K + S + offs_a = tl.arange(0, A) + top_ids = tl.zeros([A], dtype=tl.int32) + active = tl.zeros([A], dtype=tl.float32) + for kk in tl.static_range(K): + idx = tl.argmax(sel, axis=0).to(tl.int32) + raw = tl.max(tl.where(offs == idx, logits, float("-inf")), axis=0) + top_ids = tl.where(offs_a == kk, idx, top_ids) + active = tl.where(offs_a == kk, raw, active) + sel = tl.where(offs == idx, float("-inf"), sel) + if S > 0: + # Shared sink logits sit at the tail of the gate output; their expert + # ids continue after the routed range (R + j). + for jj in tl.static_range(S): + raw = tl.max(tl.where(offs == R + jj, logits, float("-inf")), axis=0) + top_ids = tl.where(offs_a == K + jj, tl.full([], R + jj, tl.int32), top_ids) + active = tl.where(offs_a == K + jj, raw, active) + + # Log-sigmoid renormalization over the K + S active logits. + abs_l = tl.abs(active) + min_l = tl.minimum(active, 0.0) + log_probs = min_l - tldevice.log1p(tldevice.exp(-abs_l)) + max_lp = tl.max(log_probs, axis=0) + exp_shifted = tldevice.exp(log_probs - max_lp) + sum_exp = tl.sum(exp_shifted, axis=0) + weights = exp_shifted / sum_exp * scale + + tl.store(ids_ptr + pid * A + offs_a, top_ids) + tl.store(weights_ptr + pid * A + offs_a, weights) + + +def inkling_gate_select( + logits: torch.Tensor, # [T, >=G] fp32 (rows may carry GEMM padding) + n_gate_experts: int, + n_routed_experts: int, + topk: int, + n_shared_experts: int, + bias: torch.Tensor | None, + route_scale: float, + global_scale: torch.Tensor | None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Sigmoid + bias + top-k + log-sigmoid renorm; returns (weights, ids).""" + assert logits.dtype == torch.float32 + tokens = logits.shape[0] + active = topk + n_shared_experts + topk_ids = torch.empty((tokens, active), dtype=torch.int32, device=logits.device) + topk_weights = torch.empty( + (tokens, active), dtype=torch.float32, device=logits.device + ) + if tokens == 0: + return topk_weights, topk_ids + _inkling_gate_select_kernel[(tokens,)]( + logits, + bias if bias is not None else logits, + global_scale if global_scale is not None else logits, + topk_ids, + topk_weights, + route_scale, + tokens, + n_gate_experts, + logits.stride(0), + n_routed_experts, + topk, + n_shared_experts, + HAS_BIAS=bias is not None, + HAS_GSCALE=global_scale is not None, + BLOCK_G=triton.next_power_of_2(n_gate_experts), + ) + return topk_weights, topk_ids + + +class InklingGate(nn.Module): + """Sigmoid gate with selection bias, log-sigmoid renorm after top-k, and + global scale (the served checkpoint's only configuration).""" + + def __init__( + self, + d_model: int, + n_routed_experts: int, + n_shared_experts: int, + experts_per_token: int, + route_scale: float, + *, + use_global_scale: bool = False, + use_gate_bias: bool = False, + ) -> None: + super().__init__() + self.n_routed_experts = n_routed_experts + self.n_shared_experts = n_shared_experts + self.n_total_experts = n_routed_experts + n_shared_experts + self.topk = experts_per_token + self.route_scale = route_scale + + padded_experts = self.n_total_experts + (-self.n_total_experts) % 8 + self.weight = Parameter( + torch.empty(padded_experts, d_model), requires_grad=False + ) + set_weight_attrs(self.weight, {"weight_loader": self._load_weight}) + if use_global_scale: + self.global_scale = Parameter( + torch.empty(1, dtype=torch.float32), requires_grad=False + ) + else: + self.global_scale = None + if use_gate_bias: + self.bias = Parameter( + torch.empty(n_routed_experts, dtype=torch.float32), + requires_grad=False, + ) + else: + self.bias = None + + @staticmethod + def _load_weight(param: Parameter, loaded_weight: torch.Tensor) -> None: + param.data.zero_() + param.data[: loaded_weight.shape[0]].copy_(loaded_weight) + + def compute_logits(self, x: torch.Tensor) -> torch.Tensor: + """fp32 gate logits [T, n_total_experts + pad] (pad columns are junk).""" + return _linear_with_fp32_out(x, self.weight) + + def select_experts( + self, gating_output: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + """Full selection: (weights, ids) of [T, K + S]. The first K entries + are the routed top-k; the S trailing entries are the sink gammas.""" + return inkling_gate_select( + gating_output, + self.n_total_experts, + self.n_routed_experts, + self.topk, + self.n_shared_experts, + self.bias, + self.route_scale, + self.global_scale, + ) + + +# --------------------------------------------------------------------------- +# MoE layer +# --------------------------------------------------------------------------- + + +def _inkling_moe_ep_size() -> int: + """EP size the FusedMoE layer will run with (mirrors + FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when + expert parallelism is enabled).""" + parallel_config = get_current_vllm_config().parallel_config + if not parallel_config.enable_expert_parallel: + return 1 + world = ( + get_tensor_model_parallel_world_size() + * get_dp_group().world_size + * get_pcp_group().world_size + ) + return world if world > 1 else 1 + + +class InklingSinkExperts(nn.Module): + """Shared "sink" experts with per-token gammas, in bf16. + + Replicated across EP ranks (every token activates every sink, so + EP-sharding them would hotspot the owning rank) and TP-sharded on the + intermediate dim so the output remains a TP-partial sum like the routed + output. The sinks are always bf16 (the checkpoint excludes every + ``shared_experts`` from quantization): the experts concatenate into two + plain dense GEMMs with the fused sink epilogue between them. + """ + + def __init__( + self, n_experts: int, d_model: int, d_mlp: int, *, prefix: str = "" + ) -> None: + super().__init__() + self.n_experts = n_experts + tp_size = get_tensor_model_parallel_world_size() + self.tp_rank = get_tensor_model_parallel_rank() + intermediate_pp = d_mlp // tp_size + self.w13_weight = Parameter( + torch.empty(n_experts, 2 * intermediate_pp, d_model), + requires_grad=False, + ) + self.w2_weight = Parameter( + torch.empty(d_model, n_experts * intermediate_pp), + requires_grad=False, + ) + self._unit: torch.Tensor | None = None + + def load_weight(self, key: str, weight: torch.Tensor) -> list[str]: + """Load one checkpoint sink tensor (stacked over the S experts).""" + if key == "w13_weight": + if weight.shape != self.w13_weight.shape: + shard = self.w13_weight.shape[1] + weight = weight.narrow(1, self.tp_rank * shard, shard) + self.w13_weight.data.copy_(weight) + return [key] + + assert key == "w2_weight" + shard = self.w2_weight.shape[1] // self.n_experts + shard_start = 0 if weight.shape[2] == shard else self.tp_rank * shard + for expert_idx, expert_weight in enumerate(weight): + local_weight = expert_weight.narrow(1, shard_start, shard) + start = expert_idx * shard + self.w2_weight.data[:, start : start + shard].copy_(local_weight) + return [key] + + def forward(self, x: torch.Tensor, gammas: torch.Tensor) -> torch.Tensor: + """``sum_e gammas[:, e] * MLP_e(x)`` (TP-partial along d_mlp).""" + from .ops import sink_silu_mul_epilogue + + # One GEMM over the experts' stacked w13 (a view), fused epilogue, + # then one GEMM whose K-reduction over the K-concatenated w2 performs + # the expert sum. + if self._unit is None or self._unit.device != x.device: + self._unit = torch.ones( + self.n_experts, dtype=torch.float32, device=x.device + ) + raw = x @ self.w13_weight.view(-1, x.shape[-1]).T # (T, S*2F) + h = sink_silu_mul_epilogue( + raw, self._unit, gammas, self._unit, self.n_experts, x.dtype + ) + return h @ self.w2_weight.T # (T, D) + + +class InklingMoE(nn.Module): + def __init__( + self, + config: InklingModelConfig, + layer_id: int, + *, + prefix: str = "", + nvfp4_config: InklingNvfp4Config | None = None, + ) -> None: + super().__init__() + # Overfit to the served checkpoint: sigmoid gate renormalized after + # top-k, shared sink experts, interleaved gate/up checkpoint rows. + assert config.gate_activation == "sigmoid" and config.norm_after_topk + assert config.n_shared_experts > 0 and config.shared_expert_sink + assert config.inference_moe_w13_interleaved + n_routed = config.n_routed_experts + n_shared = config.n_shared_experts + self.n_routed_experts = n_routed + self.gate = InklingGate( + d_model=config.hidden_size, + n_routed_experts=n_routed, + n_shared_experts=n_shared, + experts_per_token=config.num_experts_per_tok, + route_scale=config.route_scale, + use_global_scale=config.use_global_scale, + use_gate_bias=config.use_gate_bias, + ) + + moe_quant_config = None + if nvfp4_config is not None and nvfp4_config.experts_quantized(layer_id): + from vllm.model_executor.layers.quantization.modelopt import ( + ModelOptNvFp4Config, + ) + + # The Inkling checkpoint is ModelOpt NVFP4; exclusion is decided per + # layer right here, so no exclude list is needed. + moe_quant_config = ModelOptNvFp4Config( + quant_method="NVFP4", + is_checkpoint_nvfp4_serialized=True, + kv_cache_quant_algo=None, + exclude_modules=[], + group_size=nvfp4_config.group_size, + ) + + # TRTLLM MoE kernels assume equal, contiguous per-rank expert slabs + # (local_expert_offset = ep_rank * local_num_experts), so pad the + # expert count to a multiple of the EP size. A no-op for the usual + # power-of-two EP sizes (n_routed is a power of two). + num_experts = n_routed + (-n_routed) % _inkling_moe_ep_size() + + self.experts = FusedMoE( + num_experts=num_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + renormalize=False, + quant_config=moe_quant_config, + prefix=f"{prefix}.experts", + custom_routing_function=self._select_routed, + router_logits_dtype=torch.float32, + activation="silu", + ) + # The decoder layer reduce-scatters the MoE delta into the sconv + # stream itself (RS -> shard sconv -> AG); the runner must return the + # per-rank partial sum instead of all-reducing. + self.experts.moe_config.skip_final_all_reduce = True + + self._routed_sel: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None + # The sinks are always bf16; fail loudly on a checkpoint that + # quantizes them instead of silently misloading. + assert nvfp4_config is None or not nvfp4_config.shared_experts_quantized( + layer_id + ), f"layer {layer_id}: NVFP4 shared experts are not supported" + + self.sink_experts = InklingSinkExperts( + n_experts=n_shared, + d_model=config.hidden_size, + d_mlp=config.intermediate_size, + prefix=f"{prefix}.shared_experts", + ) + + # Sink chain overlaps the routed MoE call on the aux stream for + # decode-sized batches (same pattern as the runner's SharedExperts + # multi-stream overlap). The routed GEMM runs on the default stream and + # the sink chain on the aux stream, joined via these two events by + # ``maybe_execute_in_parallel``. + self._sink_stream: torch.cuda.Stream | None = aux_stream() + self._sink_events = (torch.cuda.Event(), torch.cuda.Event()) + + def _select_routed( + self, + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + ) -> tuple[torch.Tensor, torch.Tensor]: + """FusedMoE ``custom_routing_function``: the routed top-k slice of the + full (routed + sink) selection. + + forward() stashes its selection (keyed by logits identity) so the + gate select runs once per layer; the fallback covers paths where the + runner re-derives the logits (e.g. naive DP dispatch). + """ + del hidden_states, renormalize + assert topk == self.gate.topk + cached = self._routed_sel + self._routed_sel = None + if cached is not None and cached[0] is gating_output: + return cached[1], cached[2] + weights, ids = self.gate.select_experts(gating_output) + return weights[:, :topk].contiguous(), ids[:, :topk].contiguous() + + def forward(self, x: torch.Tensor) -> torch.Tensor | None: + router_logits = self.gate.compute_logits(x) + num_tokens = x.shape[0] + # One gate select per layer: the routed slice is stashed for the + # routing function inside the FusedMoE op; the sink gammas are the + # trailing columns. + k = self.gate.topk + weights, ids = self.gate.select_experts(router_logits) + self._routed_sel = ( + router_logits, + weights[:, :k].contiguous(), + ids[:, :k].contiguous(), + ) + gammas = weights[:, k:] + + out, sink_out = maybe_execute_in_parallel( + lambda: self.experts(hidden_states=x, router_logits=router_logits), + lambda: self.sink_experts(x, gammas), + self._sink_events[0], + self._sink_events[1], + self._sink_stream + if num_tokens <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD + else None, + ) + self._routed_sel = None + + return out + sink_out + + # -- weight loading ---------------------------------------------------- + + def _local_expert_slots(self) -> dict[int, int]: + """Global expert id -> local slot for this rank's expert partition.""" + manager = self.experts.routed_experts.expert_map_manager + if manager.expert_map is None: + return {g: g for g in range(manager.global_num_experts)} + emap = manager.expert_map.tolist() + return {g: slot for g, slot in enumerate(emap) if slot >= 0} + + def load_expert_weight(self, name: str, weight: torch.Tensor) -> list[str]: + """Load one checkpoint expert tensor. + + ``name`` is relative to the mlp module: ``experts.`` (routed + stack) or ``shared_experts.shared_`` (sink experts). Returns the + loaded param names (relative to this module). + """ + if name.startswith("shared_experts."): + key = name.split(".", 1)[1].replace("shared_", "", 1) + return [ + f"sink_experts.{p}" for p in self.sink_experts.load_weight(key, weight) + ] + + experts: RoutedExperts = self.experts.routed_experts + key = name.split(".", 1)[1] + + # original_shape is unused by the vLLM serving layout. + if key.endswith(".original_shape"): + return [] + if key.endswith(".input_amax"): + projection = "w13" if key.startswith("w13") else "w2" + amax = float(weight.max()) + assert math.isfinite(amax) and amax > 0, ( + f"bad {projection} input_amax: {amax}" + ) + input_scale = getattr(experts, f"{projection}_input_scale") + input_scale.data.fill_(amax / (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX)) + return [f"experts.routed_experts.{projection}_input_scale"] + + param = getattr(experts, key) + slots = self._local_expert_slots() + gids = sorted(slots) + lids = [slots[g] for g in gids] + tp_rank = experts.moe_config.moe_parallel_config.tp_rank + + if key.endswith("_scale_2"): + # Per-expert scalars, vectorized over the local experts. The + # fused w13 param carries one slot per gate/up half. + vals = weight[gids].float().to(param.device) + param.data[lids] = vals[:, None] if param.data.ndim == 2 else vals + elif key.startswith("w13"): + # Checkpoint w13 rows are interleaved [g0, u0, g1, u1, ...]; the + # fused param layout is [w1(gate); w3(up)]. The TP-local rows form + # one contiguous slab of the interleaved tensor, so upload just + # that slab (a single bounded synchronous H2D; pre-uploading whole + # untrimmed tensors pins the mmap pages of the entire checkpoint + # and OOMs the host) and de-interleave on device. + half = param.shape[1] // 2 + for gid, lid in slots.items(): + slab = weight[gid].narrow(0, tp_rank * 2 * half, 2 * half) + slab = slab.to(param.device) + param.data[lid, :half].copy_(slab[0::2]) + param.data[lid, half:].copy_(slab[1::2]) + else: + # w2: shard the packed intermediate (last) dim. + shard = param.shape[2] + for gid, lid in slots.items(): + param.data[lid].copy_(weight[gid].narrow(1, tp_rank * shard, shard)) + return [f"experts.routed_experts.{key}"] + + def finalize_load(self) -> list[str]: + """Post-load fixups for zeroed padding experts.""" + experts = self.experts.routed_experts + out: list[str] = [] + # Zero the EP-alignment padding experts (if any) so their + # (never-routed) slots hold defined values. + slots = self._local_expert_slots() + for gid in range(self.n_routed_experts, experts.global_num_experts): + lid = slots.get(gid) + if lid is None: + continue + for pname in ( + "w13_weight", + "w2_weight", + "w13_weight_scale", + "w2_weight_scale", + "w13_weight_scale_2", + "w2_weight_scale_2", + ): + p = getattr(experts, pname, None) + if p is not None: + p.data[lid].zero_() + return out diff --git a/vllm/models/inkling/nvidia/ops/__init__.py b/vllm/models/inkling/nvidia/ops/__init__.py new file mode 100644 index 0000000000..2d364fc614 --- /dev/null +++ b/vllm/models/inkling/nvidia/ops/__init__.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling kernels (NVIDIA). + +``rmsnorm`` / ``sconv`` import eagerly. The SwiGLU kernels and the FA4 +relative-attention wrapper are exposed lazily to keep this package's +import path lightweight. +""" + +from typing import TYPE_CHECKING + +from .norm import add_rmsnorm, rmsnorm +from .sconv import fused_sconv + +_LAZY_EXPORTS = { + "silu_and_mul_triton": "silu_and_mul", + "sink_silu_mul_epilogue": "silu_and_mul", + "inkling_fa4_rel_attention": "fa4_rel_attention", +} + +if TYPE_CHECKING: + from .fa4_rel_attention import inkling_fa4_rel_attention # noqa: F401 + from .silu_and_mul import ( # noqa: F401 + silu_and_mul_triton, + sink_silu_mul_epilogue, + ) + +__all__ = [ + "add_rmsnorm", + "rmsnorm", + "fused_sconv", + *sorted(_LAZY_EXPORTS), +] + + +def __getattr__(name: str): + module = _LAZY_EXPORTS.get(name) + if module is not None: + import importlib + + mod = importlib.import_module(f".{module}", __name__) + return getattr(mod, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/vllm/models/inkling/nvidia/ops/fa4_rel_attention.py b/vllm/models/inkling/nvidia/ops/fa4_rel_attention.py new file mode 100644 index 0000000000..99e12beafb --- /dev/null +++ b/vllm/models/inkling/nvidia/ops/fa4_rel_attention.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + +import torch + +from vllm.platforms import current_platform + + +def bucket_max_seqlen_q(max_seqlen_q: int) -> int: + """Round the FA4 scheduling bound up to a power of two.""" + return 1 << max(0, max_seqlen_q - 1).bit_length() + + +def inkling_fa4_num_splits( + *, + is_local: bool, + batch_size: int, + max_query_len: int, + num_heads: int, + num_kv_heads: int, + max_kv_len: int, +) -> int: + """Return the split-KV cap for FA4 with sheared relative bias.""" + capability = current_platform.get_device_capability() + if capability is not None and capability.major == 9: + return 1 + if is_local: + return 1 + + q_rows = max_query_len * (num_heads // num_kv_heads) + q_tiles = (q_rows + 255) // 256 + base_ctas = batch_size * num_kv_heads * q_tiles + # Shearing makes split/combine overhead more visible. Multi-tile causal + # prefill saturates around 64 CTAs. Batch-1 decode at very long context is + # memory-bound and uses a TP-specific cap measured through 1M KV tokens. + target_ctas = ( + 256 if q_tiles == 1 and batch_size == 1 else (128 if q_tiles == 1 else 64) + ) + max_splits = 128 + if q_tiles == 1 and batch_size == 1: + if num_kv_heads == 8: + max_splits = 16 + elif num_kv_heads == 4 or max_kv_len <= 8192: + max_splits = 32 + elif max_kv_len <= 65536: + max_splits = 64 + else: + max_splits = 128 + return max( + 1, + min(target_ctas // base_ctas, max_splits, (max_kv_len + 127) // 128), + ) + + +def inkling_fa4_rel_attention( + q: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + *, + block_table: torch.Tensor, + cache_seqlens: torch.Tensor, + cu_seqlens_q: torch.Tensor, + max_seqlen_q: int, + softmax_scale: float, + causal: bool, + window_size: tuple[int, int], + rel_extent: int, + rel_logits: torch.Tensor, + num_splits: int = 32, + out: torch.Tensor | None = None, +) -> torch.Tensor: + """Paged varlen FA4 over the bound K/V cache with the Inkling relative bias. + + ``q`` is ``(num_tokens, num_heads, head_dim)``; ``key_cache`` / ``value_cache`` + are the paged caches ``(num_blocks, block_size, num_kv_heads, head_dim)``; + ``block_table`` is the per-request page table and ``cache_seqlens`` the + per-request KV lengths (``seqused_k``). ``rel_logits`` is + ``(num_tokens, num_heads, rel_extent)``. + + The bias uses tml-fa4's sheared relative-bias layout. + """ + from vllm.third_party.tml_fa4 import flash_attn_varlen_func + + # cute uses (None, None) to mean "no window". + cute_window = (None, None) if window_size == (-1, -1) else window_size + + ret = flash_attn_varlen_func( + q=q, + k=key_cache, + v=value_cache, + cu_seqlens_q=cu_seqlens_q, + seqused_k=cache_seqlens, + max_seqlen_q=max_seqlen_q, + page_table=block_table, + softmax_scale=softmax_scale, + causal=causal, + window_size=cute_window, + num_splits=num_splits, + return_lse=False, + out=out, + rel_bias=rel_logits.contiguous(), + ) + if isinstance(ret, tuple): + return ret[0] + return ret diff --git a/vllm/models/inkling/nvidia/ops/fa4_warmup.py b/vllm/models/inkling/nvidia/ops/fa4_warmup.py new file mode 100644 index 0000000000..ff8cdb0293 --- /dev/null +++ b/vllm/models/inkling/nvidia/ops/fa4_warmup.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Startup compilation of the FA4 kernels used by Inkling.""" + +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass +from functools import partial + +import torch + +from vllm.model_executor.warmup.cutedsl_warmup import ( + CuTeDSLCompileUnit, + register_cutedsl_warmup_provider, +) + +from .fa4_rel_attention import ( + bucket_max_seqlen_q, + inkling_fa4_num_splits, + inkling_fa4_rel_attention, +) + + +@dataclass(frozen=True) +class InklingFA4WarmupConfig: + num_heads: int + num_kv_heads: int + head_dim: int + rel_extent: int + window_size: tuple[int, int] + is_local: bool + max_kv_len: int + dtype: torch.dtype + kv_dtype: torch.dtype + block_size: int + max_num_reqs: int + max_num_batched_tokens: int + + +def _num_warps_bucket(num_reqs: int) -> int: + num_warps = min((num_reqs + 30) // 31, 32) + return 1 << (num_warps - 1).bit_length() + + +def _compile(config: InklingFA4WarmupConfig, max_seqlen_q: int, num_reqs: int) -> None: + from torch._subclasses.fake_tensor import FakeTensorMode + + num_splits = inkling_fa4_num_splits( + is_local=config.is_local, + batch_size=num_reqs, + max_query_len=max_seqlen_q, + num_heads=config.num_heads, + num_kv_heads=config.num_kv_heads, + max_kv_len=config.max_kv_len, + ) + with FakeTensorMode(): + device = torch.accelerator.current_accelerator() + total_q = max_seqlen_q + num_reqs - 1 + q = torch.empty( + total_q, + config.num_heads, + config.head_dim, + dtype=config.dtype, + device=device, + ) + kv = torch.empty( + 1, + 2, + config.block_size, + config.num_kv_heads, + config.head_dim, + dtype=config.kv_dtype, + device=device, + ) + key_cache, value_cache = kv.unbind(1) + inkling_fa4_rel_attention( + q, + key_cache, + value_cache, + block_table=torch.empty(num_reqs, 1, dtype=torch.int32, device=device), + cache_seqlens=torch.empty(num_reqs, dtype=torch.int32, device=device), + cu_seqlens_q=torch.empty(num_reqs + 1, dtype=torch.int32, device=device), + max_seqlen_q=max_seqlen_q, + softmax_scale=1.0 / config.head_dim, + causal=True, + window_size=config.window_size, + rel_extent=config.rel_extent, + rel_logits=torch.empty( + total_q, + config.num_heads, + config.rel_extent, + dtype=config.dtype, + device=device, + ), + num_splits=num_splits, + out=torch.empty_like(q), + ) + + +def _iter_compile_units( + config: InklingFA4WarmupConfig, +) -> Iterator[CuTeDSLCompileUnit]: + max_bucket = bucket_max_seqlen_q(config.max_num_batched_tokens) + max_seqlen_q = 1 + while max_seqlen_q <= max_bucket: + min_query_len = max_seqlen_q // 2 + 1 + max_num_reqs = min( + config.max_num_reqs, + config.max_num_batched_tokens - min_query_len + 1, + ) + seen: set[tuple[int, int, int | None, bool]] = set() + for num_reqs in range(1, max_num_reqs + 1): + num_splits = inkling_fa4_num_splits( + is_local=config.is_local, + batch_size=num_reqs, + max_query_len=max_seqlen_q, + num_heads=config.num_heads, + num_kv_heads=config.num_kv_heads, + max_kv_len=config.max_kv_len, + ) + key = ( + max_seqlen_q, + num_splits, + _num_warps_bucket(num_reqs) if num_splits > 1 else None, + num_reqs > 1024, + ) + if key in seen: + continue + seen.add(key) + yield CuTeDSLCompileUnit( + name="inkling_fa4", + key=("inkling_fa4", config, key), + compile=partial(_compile, config, max_seqlen_q, num_reqs), + ) + max_seqlen_q *= 2 + + +class _WarmupProvider: + def __init__(self) -> None: + self.configs: set[InklingFA4WarmupConfig] = set() + + def get_cutedsl_warmup_compile_units(self) -> tuple[CuTeDSLCompileUnit, ...]: + return tuple( + unit for config in self.configs for unit in _iter_compile_units(config) + ) + + +_PROVIDER = _WarmupProvider() + + +def register_fa4_warmup(config: InklingFA4WarmupConfig) -> None: + _PROVIDER.configs.add(config) + register_cutedsl_warmup_provider(_PROVIDER) diff --git a/vllm/models/inkling/nvidia/ops/lamport.py b/vllm/models/inkling/nvidia/ops/lamport.py new file mode 100644 index 0000000000..c02290dc48 --- /dev/null +++ b/vllm/models/inkling/nvidia/ops/lamport.py @@ -0,0 +1,766 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Deadlock-free fused RS + short-conv + AG + residual + RMSNorm. + +The public integration surface is ``LamportRSConv.rs_sconv_ag_add_norm``. + +Liveness +-------- +Large grids are deliberately split at the two communication dependencies: + + 1. ``_publish_input_kernel`` only publishes rank partials. + 2. ``_reduce_insert_kernel`` waits, reduces, and inserts the local shard. + 3. ``_sconv_publish_kernel`` only computes and publishes the local result. + 4. ``_gather_norm_kernel`` waits, gathers, and normalizes. + +Without PDL, CUDA stream order completes a producer before its consumer. With +PDL, every producer CTA posts all peer stores before triggering its dependent, +and the consumer executes ``gdc_wait`` before polling. A consumer therefore +waits only for stores from a producer that is already running (or complete) on +another GPU. It never waits for another block in its own grid. Consequently +spinning consumers cannot occupy resources needed by any producer, and the +wait-for graph has no cycle. The proof is independent of grid size, block +dispatch order, and occupancy. + +For one token, the first three phases use eight independent channel slices to +expose enough CTA parallelism for decode latency. Each slice has exclusive +ownership of its cache and Lamport columns; the gather/RMSNorm phase retains +one CTA per token so no cross-CTA reduction or completion counter is needed. + +Immediate buffer reuse (including replay of a captured CUDA graph) is also +safe. Rank R cannot republish input for call n+1 until its gather for n has +finished; that gather waited for owner O's output, which O publishes only +after consuming R's input for n. Likewise, R cannot republish output for n+1 +until its reduction for n+1 has observed destination D's input; D publishes +that input only after its gather consumed R's output for n. Thus every prior +read happens-before a same-slot rewrite. Three generations reduce incidental +coupling for ordinary launches, but correctness does not depend on rotation. + +The payload itself is the Lamport flag. A 32-bit store publishes two bf16s +atomically; 0x80008000 (two negative zeroes) denotes an empty pair. Real +negative zeroes are changed to positive zero before publication. Consumers +use volatile 32-bit loads and restore the sentinel after consuming a slot. +""" + +from __future__ import annotations + +import os + +import torch + +from vllm.distributed import get_tp_group +from vllm.distributed.parallel_state import in_the_same_node_as +from vllm.logger import init_logger +from vllm.triton_utils import HAS_TRITON, tl, triton + +logger = init_logger(__name__) + +_MAX_TOKENS = 16384 +_EMPTY_PAIR = tl.constexpr(0x80008000) if HAS_TRITON else 0x80008000 + + +@triton.jit +def _pack_bf16_pairs(values): + """Pack bf16 values into atomic u32 pairs and reserve negative zero.""" + lo, hi = tl.split(values.reshape([values.shape[0] // 2, 2])) + lo = lo.to(tl.uint16, bitcast=True) + hi = hi.to(tl.uint16, bitcast=True) + lo = tl.where(lo == 0x8000, 0, lo).to(tl.uint32) + hi = tl.where(hi == 0x8000, 0, hi).to(tl.uint32) + return lo | (hi << 16) + + +@triton.jit +def _unpack_bf16_pairs(values): + lo = (values & 0xFFFF).to(tl.uint16).to(tl.bfloat16, bitcast=True) + hi = (values >> 16).to(tl.uint16).to(tl.bfloat16, bitcast=True) + return tl.interleave(lo, hi) + + +@triton.jit +def _wait_pairs(ptr, offsets, mask): + values = tl.load(ptr + offsets, mask=mask, other=0, volatile=True) + while tl.max(tl.where(mask & (values == _EMPTY_PAIR), 1, 0)) != 0: + values = tl.load(ptr + offsets, mask=mask, other=0, volatile=True) + return values + + +@triton.jit +def _publish_input_kernel( + stage_ptr, + peer_ptrs, + peer_offset_u32, + stride_stage_t, + C: tl.constexpr, + CS: tl.constexpr, + CS_P2: tl.constexpr, + SPLITS: tl.constexpr, + RANK: tl.constexpr, + WORLD: tl.constexpr, + USE_PDL: tl.constexpr, + launch_pdl: tl.constexpr, +): + """Publish this rank's full partial row into every shard owner's slots.""" + token = tl.program_id(0).to(tl.int64) + split = tl.program_id(1).to(tl.int64) + CSS: tl.constexpr = CS // SPLITS + pair = tl.arange(0, CS_P2 // 2) + pair_mask = pair < CSS // 2 + elem = tl.arange(0, CS_P2) + elem_mask = elem < CSS + ptrs = peer_ptrs.to(tl.pointer_type(tl.uint64)) + # Address generation is independent of the preceding stream kernel. The + # acquire remains before stage/peer loads, which may consume its writes. + if USE_PDL: + tl.extra.cuda.gdc_wait() + + for owner in tl.static_range(WORLD): + values = tl.load( + stage_ptr + token * stride_stage_t + owner * CS + split * CSS + elem, + mask=elem_mask, + other=0.0, + ) + packed = _pack_bf16_pairs(values) + base = tl.load(ptrs + owner).to(tl.pointer_type(tl.uint32)) + dst = (token * WORLD + RANK) * (CS // 2) + split * (CSS // 2) + pair + tl.store(base + peer_offset_u32 + dst, packed, mask=pair_mask) + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + + +@triton.jit +def _reduce_insert_kernel( + input_peer_ptrs, + input_peer_offset_u32, + cache_ptr, + slot_ptr, + stride_cache_block, + stride_cache_head, + stride_cache_token, + stride_cache_dim, + block_size, + C: tl.constexpr, + CS: tl.constexpr, + CS_P2: tl.constexpr, + SPLITS: tl.constexpr, + HEAD_SIZE: tl.constexpr, + CACHE_OFFSET: tl.constexpr, + RANK: tl.constexpr, + WORLD: tl.constexpr, + USE_PDL: tl.constexpr, + launch_pdl: tl.constexpr, +): + """Consume all partials for this rank and insert the reduced cache row.""" + token = tl.program_id(0).to(tl.int64) + split = tl.program_id(1).to(tl.int64) + CSS: tl.constexpr = CS // SPLITS + source = tl.arange(0, WORLD) + pair = tl.arange(0, CS_P2 // 2) + pair_mask = pair < CSS // 2 + offsets = ( + (token * WORLD + source)[:, None] * (CS // 2) + + split * (CSS // 2) + + pair[None, :] + ) + mask = tl.full([WORLD], True, tl.int1)[:, None] & pair_mask[None, :] + input_ptrs = input_peer_ptrs.to(tl.pointer_type(tl.uint64)) + input_u32 = tl.load(input_ptrs + RANK).to(tl.pointer_type(tl.uint32)) + input_u32 += input_peer_offset_u32 + # Slot metadata and the cache destination do not depend on input publish. + slot = tl.load(slot_ptr + token) + valid = slot >= 0 + channel = tl.arange(0, CS_P2) + channel_mask = channel < CSS + global_channel = split * CSS + channel + head = tl.minimum(global_channel // HEAD_SIZE, CS // HEAD_SIZE - 1) + dim = CACHE_OFFSET + global_channel % HEAD_SIZE + safe_slot = tl.maximum(slot, 0).to(tl.int64) + dst = ( + cache_ptr + + (safe_slot // block_size) * stride_cache_block + + head * stride_cache_head + + (safe_slot % block_size) * stride_cache_token + + dim * stride_cache_dim + ) + if USE_PDL: + tl.extra.cuda.gdc_wait() + packed = _wait_pairs(input_u32, offsets, mask) + + lo = (packed & 0xFFFF).to(tl.uint16).to(tl.bfloat16, bitcast=True) + hi = (packed >> 16).to(tl.uint16).to(tl.bfloat16, bitcast=True) + reduced = tl.interleave( + tl.sum(lo.to(tl.float32), axis=0).to(tl.bfloat16), + tl.sum(hi.to(tl.float32), axis=0).to(tl.bfloat16), + ) + tl.store( + input_u32 + offsets, + tl.full([WORLD, CS_P2 // 2], _EMPTY_PAIR, tl.uint32), + mask=mask, + ) + + tl.store(dst, reduced, mask=valid & channel_mask) + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + + +@triton.jit +def _sconv_publish_kernel( + peer_ptrs, + peer_offset_u32, + residual_ptr, + weight_ptr, + cache_ptr, + position_ptr, + sequence_ptr, + slot_ptr, + block_table_ptr, + stride_residual_t, + stride_cache_block, + stride_cache_head, + stride_cache_token, + stride_cache_dim, + stride_block_table_r, + max_blocks, + block_size, + C: tl.constexpr, + CS: tl.constexpr, + CS_P2: tl.constexpr, + SPLITS: tl.constexpr, + HEAD_SIZE: tl.constexpr, + CACHE_OFFSET: tl.constexpr, + RANK: tl.constexpr, + WORLD: tl.constexpr, + WINDOW: tl.constexpr, + USE_PDL: tl.constexpr, + launch_pdl: tl.constexpr, +): + """Compute this rank's shard, then publish it to every rank.""" + tl.static_assert(WINDOW == 4) + token = tl.program_id(0).to(tl.int64) + split = tl.program_id(1).to(tl.int64) + CSS: tl.constexpr = CS // SPLITS + channel = tl.arange(0, CS_P2) + channel_mask = channel < CSS + global_channel = split * CSS + channel + head = tl.minimum(global_channel // HEAD_SIZE, CS // HEAD_SIZE - 1) + dim = CACHE_OFFSET + global_channel % HEAD_SIZE + slot = tl.load(slot_ptr + token) + valid = slot >= 0 + position = tl.load(position_ptr + token) + sequence = tl.load(sequence_ptr + token) + safe_slot = tl.maximum(slot, 0).to(tl.int64) + own_ptr = ( + cache_ptr + + (safe_slot // block_size) * stride_cache_block + + head * stride_cache_head + + (safe_slot % block_size) * stride_cache_token + + dim * stride_cache_dim + ) + # These loads were made visible before reduce/insert could signal us and + # are independent of its cache store. Hoisting them gives PDL useful work + # to overlap while retaining the acquire before every cache read. + residual = tl.load( + residual_ptr + token * stride_residual_t + RANK * CS + global_channel, + mask=channel_mask, + other=0.0, + ) + weight0 = tl.load( + weight_ptr + global_channel * WINDOW, + mask=channel_mask, + other=0.0, + ).to(tl.float32) + weight1 = tl.load( + weight_ptr + global_channel * WINDOW + 1, + mask=channel_mask, + other=0.0, + ).to(tl.float32) + weight2 = tl.load( + weight_ptr + global_channel * WINDOW + 2, + mask=channel_mask, + other=0.0, + ).to(tl.float32) + weight3 = tl.load( + weight_ptr + global_channel * WINDOW + 3, + mask=channel_mask, + other=0.0, + ).to(tl.float32) + ptrs = peer_ptrs.to(tl.pointer_type(tl.uint64)) + if USE_PDL: + tl.extra.cuda.gdc_wait() + current = tl.load(own_ptr, mask=valid & channel_mask, other=0.0) + + conv = tl.zeros([CS_P2], tl.float32) + for tap_idx in tl.static_range(WINDOW): + source_position = position - (WINDOW - 1) + tap_idx + take = valid & (source_position >= 0) + if tap_idx == WINDOW - 1: + value = tl.where(take, current.to(tl.float32), 0.0) + else: + safe_position = tl.maximum(source_position, 0) + logical_block = tl.minimum(safe_position // block_size, max_blocks - 1) + physical_block = tl.load( + block_table_ptr + sequence * stride_block_table_r + logical_block, + mask=take, + other=0, + ).to(tl.int64) + source_ptr = ( + cache_ptr + + physical_block * stride_cache_block + + head * stride_cache_head + + (safe_position % block_size) * stride_cache_token + + dim * stride_cache_dim + ) + cached = tl.load(source_ptr, mask=take & channel_mask, other=0.0) + value = tl.where(take, cached.to(tl.float32), 0.0) + if tap_idx == 0: + weight = weight0 + elif tap_idx == 1: + weight = weight1 + elif tap_idx == 2: + weight = weight2 + else: + weight = weight3 + conv += value * weight + + # Preserve both bf16 rounding points of the original sublayer. + short_conv_with_skip = (conv + current.to(tl.float32)).to(tl.bfloat16) + output = (residual.to(tl.float32) + short_conv_with_skip.to(tl.float32)).to( + tl.bfloat16 + ) + + pair = tl.arange(0, CS_P2 // 2) + pair_mask = pair < CSS // 2 + packed = _pack_bf16_pairs(output) + row_offset = token * (C // 2) + RANK * (CS // 2) + split * (CSS // 2) + pair + + for destination in tl.static_range(WORLD): + base = tl.load(ptrs + destination).to(tl.pointer_type(tl.uint32)) + tl.store(base + peer_offset_u32 + row_offset, packed, mask=pair_mask) + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + + +@triton.jit +def _gather_norm_kernel( + output_peer_ptrs, + output_peer_offset_u32, + norm_weight_ptr, + normed_ptr, + residual_out_ptr, + eps, + stride_output_t, + C: tl.constexpr, + C_P2: tl.constexpr, + RANK: tl.constexpr, + HAS_NORM: tl.constexpr, + USE_PDL: tl.constexpr, + launch_pdl: tl.constexpr, +): + """Consume a complete row; one CTA owns all outputs for one token.""" + token = tl.program_id(0).to(tl.int64) + pair = tl.arange(0, C_P2 // 2) + pair_mask = pair < C // 2 + output_ptrs = output_peer_ptrs.to(tl.pointer_type(tl.uint64)) + output_u32 = tl.load(output_ptrs + RANK).to(tl.pointer_type(tl.uint32)) + output_u32 += output_peer_offset_u32 + offsets = token * (C // 2) + pair + channel = tl.arange(0, C_P2) + channel_mask = channel < C + # Gamma is independent of the preceding sconv publication. Keep the + # acquire immediately before polling the Lamport output slots. + if HAS_NORM: + weight = tl.load(norm_weight_ptr + channel, mask=channel_mask, other=0.0) + if USE_PDL: + tl.extra.cuda.gdc_wait() + packed = _wait_pairs(output_u32, offsets, pair_mask) + row = _unpack_bf16_pairs(packed) + tl.store( + residual_out_ptr + token * stride_output_t + channel, row, mask=channel_mask + ) + if HAS_NORM: + row_f32 = tl.where(channel_mask, row.to(tl.float32), 0.0) + inv_rms = tl.rsqrt(tl.sum(row_f32 * row_f32, axis=0) / C + eps) + tl.store( + normed_ptr + token * stride_output_t + channel, + (row_f32 * inv_rms * weight.to(tl.float32)).to(tl.bfloat16), + mask=channel_mask, + ) + tl.store( + output_u32 + offsets, + tl.full([C_P2 // 2], _EMPTY_PAIR, tl.uint32), + mask=pair_mask, + ) + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + + +@triton.jit +def _validate_lamport_init_kernel( + input_peer_ptrs, + output_peer_ptrs, + bad_ptr, + num_pairs, + RANK: tl.constexpr, + BLOCK: tl.constexpr, +): + """Validate both complete local allocations through their fabric pointers.""" + offsets = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK) + mask = offsets < num_pairs + ptrs_in = input_peer_ptrs.to(tl.pointer_type(tl.uint64)) + ptrs_out = output_peer_ptrs.to(tl.pointer_type(tl.uint64)) + local_in = tl.load(ptrs_in + RANK).to(tl.pointer_type(tl.uint32)) + local_out = tl.load(ptrs_out + RANK).to(tl.pointer_type(tl.uint32)) + value_in = tl.load(local_in + offsets, mask=mask, other=_EMPTY_PAIR) + value_out = tl.load(local_out + offsets, mask=mask, other=_EMPTY_PAIR) + bad = tl.max( + tl.where(mask & ((value_in != _EMPTY_PAIR) | (value_out != _EMPTY_PAIR)), 1, 0) + ) + if bad != 0: + tl.atomic_max(bad_ptr, 1) + + +class LamportRSConv: + """Persistent symmetric buffers for one TP group.""" + + def _initialize_lamport_buffers(self) -> None: + """Arm every slot and collectively verify the exact sentinel bits. + + Do not replace this with a zero-fill: Lamport readiness distinguishes + the bf16 bit pattern for -0.0 (0x8000) from every published payload. + The validation is deliberately collective so one rank cannot enter the + first polling kernel while another rank still has an unarmed buffer. + """ + if not self.buf_in.is_contiguous() or not self.buf_out.is_contiguous(): + raise RuntimeError("Lamport symmetric buffers must be contiguous") + if self.buf_in.numel() % 2 or self.buf_out.numel() % 2: + raise RuntimeError("Lamport buffers must contain whole uint32 pairs") + + # Fill through int16 so each bf16 lane receives the exact -0.0 bits. + # This covers all three generations and all max-token slots, including + # slots that a smaller first invocation does not touch. + self.buf_in.view(torch.int16).fill_(-0x8000) + self.buf_out.view(torch.int16).fill_(-0x8000) + torch.accelerator.synchronize(self.device) + self.tp.barrier() + + # Scan the complete local allocations. Since every rank performs the + # scan and participates in MAX, success proves every symmetric backing + # allocation was armed before any rank is allowed to use generation 0. + bad = torch.logical_or( + self.buf_in.view(torch.int16).ne(-0x8000).any(), + self.buf_out.view(torch.int16).ne(-0x8000).any(), + ).to(dtype=torch.float32) + bad = self.tp.all_reduce(bad) + if int(bad.item()) != 0: + raise RuntimeError("Lamport sentinel initialization failed on a TP rank") + self.tp.barrier() + + def _initialize_mnnvl_buffers(self) -> None: + """Initialize and validate FlashInfer fabric-mapped Lamport storage.""" + self._mnnvl_input_handle.lamport_initialize(self.rank, torch.bfloat16) + self._mnnvl_output_handle.lamport_initialize(self.rank, torch.bfloat16) + torch.accelerator.synchronize(self.device) + self.tp.barrier() + + bad = torch.zeros((), dtype=torch.int32, device=self.device) + num_pairs = self.num_buffers * self.max_tokens * self.hidden_size // 2 + _validate_lamport_init_kernel[(triton.cdiv(num_pairs, 256),)]( + self.input_peer_ptrs, + self.output_peer_ptrs, + bad, + num_pairs, + RANK=self.rank, + BLOCK=256, + num_warps=4, + ) + bad = self.tp.all_reduce(bad.to(torch.float32)) + if int(bad.item()) != 0: + raise RuntimeError("MNNVL Lamport sentinel initialization failed") + self.tp.barrier() + + def __init__( + self, hidden_size: int, window_size: int, max_tokens: int = _MAX_TOKENS + ) -> None: + import torch.distributed._symmetric_memory as symm_mem + + tp = get_tp_group() + self.tp = tp + self.group = tp.device_group + self.world_size = tp.world_size + self.rank = tp.rank_in_group + self.device = torch.device(tp.device) + if self.world_size not in (2, 4, 8): + raise ValueError(f"TP world size must be 2, 4, or 8, got {self.world_size}") + if hidden_size % (2 * self.world_size) != 0: + raise ValueError("hidden size must produce an even shard on every rank") + if window_size != 4: + raise ValueError(f"short-conv window size must be 4, got {window_size}") + if max_tokens < 1 or max_tokens > _MAX_TOKENS: + raise ValueError(f"max_tokens must be in [1, {_MAX_TOKENS}]") + + is_cross_node = not all(in_the_same_node_as(tp.cpu_group)) + self.hidden_size = hidden_size + self.window_size = window_size + self.max_tokens = max_tokens + self.shard_size = hidden_size // self.world_size + # Three generations follow FlashInfer's Lamport layout. A generation + # is reused only after two intervening collective calls. + self.num_buffers = 3 + self.input_generation_bytes = max_tokens * hidden_size * 2 + self.output_generation_bytes = max_tokens * hidden_size * 2 + self.use_pdl = torch.cuda.get_device_capability(self.device)[0] >= 9 + if is_cross_node: + try: + from flashinfer.comm.mnnvl import ( + McastGPUBuffer, + TorchDistBackend, + is_mnnvl_fabric_supported, + ) + except ImportError as error: + raise RuntimeError( + "cross-node TP requires FlashInfer MNNVL support" + ) from error + + local_supported = int( + is_mnnvl_fabric_supported(torch.accelerator.current_device_index()) + ) + unsupported = torch.tensor( + 1 - local_supported, dtype=torch.float32, device=self.device + ) + unsupported = tp.all_reduce(unsupported) + if int(unsupported.item()) != 0: + raise RuntimeError("cross-node TP is supported only on MNNVL fabric") + + comm_backend = TorchDistBackend(self.group) + allocation_bytes = self.num_buffers * max_tokens * hidden_size * 2 + self._mnnvl_input_handle = McastGPUBuffer( + allocation_bytes, + self.world_size, + self.rank, + self.device, + comm_backend, + ) + self._mnnvl_output_handle = McastGPUBuffer( + allocation_bytes, + self.world_size, + self.rank, + self.device, + comm_backend, + ) + self.input_peer_ptrs = self._mnnvl_input_handle.get_buffer_ptrs_dev() + self.output_peer_ptrs = self._mnnvl_output_handle.get_buffer_ptrs_dev() + self._initialize_mnnvl_buffers() + logger.info("using FlashInfer fabric-mapped MNNVL Lamport buffers") + else: + self.buf_in = symm_mem.empty( + self.num_buffers, + max_tokens, + self.world_size, + self.shard_size, + dtype=torch.bfloat16, + device=self.device, + ) + self.buf_out = symm_mem.empty( + self.num_buffers, + max_tokens, + hidden_size, + dtype=torch.bfloat16, + device=self.device, + ) + group_name = self.group.group_name + input_handle = symm_mem.rendezvous(self.buf_in, group_name) + output_handle = symm_mem.rendezvous(self.buf_out, group_name) + self.input_peer_ptrs = input_handle.buffer_ptrs_dev + self.output_peer_ptrs = output_handle.buffer_ptrs_dev + self._input_handle = input_handle + self._output_handle = output_handle + self._initialize_lamport_buffers() + self.generation = 0 + + def usable(self, num_tokens: int) -> bool: + return 0 < num_tokens <= self.max_tokens + + def rs_sconv_ag_add_norm( + self, + input_tensor: torch.Tensor, + residual: torch.Tensor, + conv_weight: torch.Tensor, + norm_weight: torch.Tensor | None, + eps: float, + cache: torch.Tensor, + positions: torch.Tensor, + block_table: torch.Tensor, + seq_idx: torch.Tensor, + slot_mapping: torch.Tensor, + off_s: int, + ws: int, + block_size: int, + ) -> tuple[torch.Tensor | None, torch.Tensor]: + """Return ``(normed | None, new_residual)``, both shaped ``[T, 6144]``.""" + tokens, hidden_size = residual.shape + if not self.usable(tokens): + raise ValueError(f"num_tokens must be in [1, {self.max_tokens}]") + if hidden_size != self.hidden_size or residual.dtype != torch.bfloat16: + raise ValueError("residual must be bf16 [T, 6144]") + if ( + input_tensor.shape != residual.shape + or input_tensor.dtype != torch.bfloat16 + or input_tensor.stride(1) != 1 + ): + raise ValueError("input_tensor must be channel-contiguous bf16 [T, 6144]") + shard_size = hidden_size // self.world_size + if conv_weight.shape != (shard_size, self.window_size): + raise ValueError( + f"conv_weight must have shape [{shard_size}, {self.window_size}]" + ) + if ( + conv_weight.dtype != torch.bfloat16 + or conv_weight.stride(0) != self.window_size + ): + raise ValueError("conv_weight must be contiguous bf16") + if norm_weight is not None and ( + norm_weight.shape != (hidden_size,) or norm_weight.dtype != torch.bfloat16 + ): + raise ValueError("norm_weight must be bf16 [6144] or None") + if cache.dtype != torch.bfloat16 or cache.ndim != 4: + raise ValueError("cache must be a 4-D bf16 tensor") + if shard_size % ws != 0 or cache.shape[1] != shard_size // ws: + raise ValueError("cache head layout is inconsistent with ws") + if off_s < 0 or off_s + ws > cache.shape[3]: + raise ValueError("cache channel offset is out of bounds") + + index = self.generation + input_offset = index * self.input_generation_bytes // 4 + output_offset = index * self.output_generation_bytes // 4 + normed = torch.empty_like(residual) if norm_weight is not None else None + residual_out = torch.empty_like(residual) + phase_splits = 8 if tokens == 1 and shard_size % 8 == 0 else 1 + phase_tile_p2 = triton.next_power_of_2(shard_size // phase_splits) + phase_grid = (tokens, phase_splits) + # Wide CTAs win before the grid saturates; smaller CTAs reduce register + # pressure once high-throughput batches provide enough parallelism. + if 128 <= tokens <= 2048: + phase_warps = 16 + elif tokens > 2048: + phase_warps = 8 + else: + phase_warps = 4 + gather_warps = 4 if tokens >= 256 else 8 + _publish_input_kernel[phase_grid]( + input_tensor, + self.input_peer_ptrs, + input_offset, + input_tensor.stride(0), + C=hidden_size, + CS=shard_size, + CS_P2=phase_tile_p2, + SPLITS=phase_splits, + RANK=self.rank, + WORLD=self.world_size, + USE_PDL=self.use_pdl, + launch_pdl=self.use_pdl, + num_warps=phase_warps, + ) + _reduce_insert_kernel[phase_grid]( + self.input_peer_ptrs, + input_offset, + cache, + slot_mapping, + cache.stride(0), + cache.stride(1), + cache.stride(2), + cache.stride(3), + block_size, + C=hidden_size, + CS=shard_size, + CS_P2=phase_tile_p2, + SPLITS=phase_splits, + HEAD_SIZE=ws, + CACHE_OFFSET=off_s, + RANK=self.rank, + WORLD=self.world_size, + USE_PDL=self.use_pdl, + launch_pdl=self.use_pdl, + num_warps=phase_warps, + ) + _sconv_publish_kernel[phase_grid]( + self.output_peer_ptrs, + output_offset, + residual, + conv_weight, + cache, + positions, + seq_idx, + slot_mapping, + block_table, + residual.stride(0), + cache.stride(0), + cache.stride(1), + cache.stride(2), + cache.stride(3), + block_table.stride(0), + block_table.shape[1], + block_size, + C=hidden_size, + CS=shard_size, + CS_P2=phase_tile_p2, + SPLITS=phase_splits, + HEAD_SIZE=ws, + CACHE_OFFSET=off_s, + RANK=self.rank, + WORLD=self.world_size, + WINDOW=self.window_size, + USE_PDL=self.use_pdl, + launch_pdl=self.use_pdl, + num_warps=phase_warps, + ) + _gather_norm_kernel[(tokens,)]( + self.output_peer_ptrs, + output_offset, + norm_weight if norm_weight is not None else residual, + normed if normed is not None else residual_out, + residual_out, + eps, + residual.stride(0), + C=hidden_size, + C_P2=triton.next_power_of_2(hidden_size), + RANK=self.rank, + HAS_NORM=norm_weight is not None, + USE_PDL=self.use_pdl, + launch_pdl=self.use_pdl, + num_warps=gather_warps, + ) + self.generation = (index + 1) % self.num_buffers + return normed, residual_out + + +_STATE: LamportRSConv | None = None +_STATE_FAILED = False + + +def initialize_lamport_rs_conv( + hidden_size: int, window_size: int, max_num_batched_tokens: int +) -> None: + """Collectively initialize the TP-group state during model construction.""" + global _STATE, _STATE_FAILED + if _STATE is not None: + if _STATE.hidden_size != hidden_size or _STATE.window_size != window_size: + raise RuntimeError("all Lamport users must share hidden and window sizes") + return + if _STATE_FAILED or os.environ.get("LAMPORT_RS_SCONV", "1") == "0": + return + try: + max_tokens = min(_MAX_TOKENS, max_num_batched_tokens) + _STATE = LamportRSConv(hidden_size, window_size, max_tokens=max_tokens) + except Exception: + _STATE_FAILED = True + logger.exception("fused collective unavailable; use the NCCL fallback") + + +def get_lamport_rs_conv(hidden_size: int, window_size: int) -> LamportRSConv | None: + """Return the state initialized with the model, or ``None`` for fallback.""" + if _STATE is not None and ( + _STATE.hidden_size != hidden_size or _STATE.window_size != window_size + ): + raise RuntimeError("all Lamport users must share hidden and window sizes") + return _STATE diff --git a/vllm/models/inkling/nvidia/ops/mm_towers.py b/vllm/models/inkling/nvidia/ops/mm_towers.py new file mode 100644 index 0000000000..021cf5274b --- /dev/null +++ b/vllm/models/inkling/nvidia/ops/mm_towers.py @@ -0,0 +1,190 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Fused CUDA kernels for the Inkling vision/audio towers. + +Both kernels keep the reference paths' fp32 accumulation and per-op bf16 +rounding points (native ``rms_norm`` / ``F.gelu``); outputs are frequently +bit-identical and otherwise differ by 1-2 bf16 ulps from reduction-order +(real-checkpoint-weight cosine vs reference > 0.9999998). +""" + +from __future__ import annotations + +import torch + +from vllm.triton_utils import tl, tldevice, triton + +from .norm import _get_num_warps_from_block_size + + +@triton.jit +def _dmel_embed_sum_norm_kernel( + idx_ptr, # [T, NB] int32 dMel bin indices (values in [0, VOCAB)) + w_ptr, # [NB * VOCAB, D] bf16 embedding table + norm_w_ptr, # [D] (unused if HAS_NORM=False) + out_ptr, # [T, D] bf16 + eps, + D, + stride_idx_t, + NB: tl.constexpr, + VOCAB: tl.constexpr, + D_P2: tl.constexpr, + HAS_NORM: tl.constexpr, +): + t = tl.program_id(0).to(tl.int64) + offs = tl.arange(0, D_P2) + mask = offs < D + # One embedding row per mel bin (bin b uses table rows [b*VOCAB, (b+1)*VOCAB)), + # summed in fp32 (matches torch's fp32-accumulated bf16 .sum()). + acc = tl.zeros([D_P2], dtype=tl.float32) + for b in tl.static_range(NB): + v = tl.load(idx_ptr + t * stride_idx_t + b) + row = (b * VOCAB + v).to(tl.int64) + acc += tl.load(w_ptr + row * D + offs, mask=mask, other=0.0).to(tl.float32) + h = acc.to(tl.bfloat16) + if HAS_NORM: + # Match ir.ops.rms_norm: fp32 variance/normalize, then a single-rounded + # bf16 multiply with the bf16 weight. + x32 = h.to(tl.float32) + var = tl.sum(tl.where(mask, x32 * x32, 0.0), axis=0) / D + xn = (x32 * tl.math.rsqrt(var + eps)).to(tl.bfloat16) + w = tl.load(norm_w_ptr + offs, mask=mask, other=0.0) + h = xn * w + tl.store(out_ptr + t * D + offs, h, mask=mask) + + +def dmel_embed_sum_norm( + dmel_idx: torch.Tensor, # [T, NB] int32 + weight: torch.Tensor, # [NB * VOCAB, D] bf16 + norm_weight: torch.Tensor | None, + eps: float, +) -> torch.Tensor: + """``rmsnorm(sum_b weight[b * VOCAB + idx[:, b]])`` in one launch (no + [T, NB, D] intermediate).""" + T, nb = dmel_idx.shape + D = weight.shape[1] + assert weight.shape[0] % nb == 0 + vocab = weight.shape[0] // nb + out = torch.empty((T, D), dtype=weight.dtype, device=weight.device) + if T == 0: + return out + d_p2 = triton.next_power_of_2(D) + _dmel_embed_sum_norm_kernel[(T,)]( + dmel_idx, + weight, + norm_weight if norm_weight is not None else weight, + out, + eps, + D, + dmel_idx.stride(0), + NB=nb, + VOCAB=vocab, + D_P2=d_p2, + HAS_NORM=norm_weight is not None, + # Swept on GB200: 8 warps beats the block-size heuristic's 16 by ~4% + # at large T (the 80-row gather chain is latency- not lane-bound). + num_warps=8, + ) + return out + + +@triton.jit +def _rmsnorm_gelu_kernel( + x_ptr, # [R, D] bf16 + w_ptr, # [D] + out_ptr, # [R, D] bf16 (or the folded layout when FOLD) + eps, + R, + D, + D_P2: tl.constexpr, + BLOCK_M: tl.constexpr, # rows per block (>1 for small D) + HAS_GELU: tl.constexpr, + FOLD: tl.constexpr, + # fold geometry: input rows index [N, T, H, W]; the store scatters each + # row to (out_row, slot) of fold_timespace_to_depth's output layout. + FT: tl.constexpr, + FH: tl.constexpr, + FW: tl.constexpr, + TF: tl.constexpr, + HF: tl.constexpr, +): + pid = tl.program_id(0).to(tl.int64) + rows = pid * BLOCK_M + tl.arange(0, BLOCK_M) + rmask = rows < R + offs = tl.arange(0, D_P2) + mask = rmask[:, None] & (offs < D)[None, :] + x32 = tl.load(x_ptr + rows[:, None] * D + offs[None, :], mask=mask, other=0.0).to( + tl.float32 + ) + var = tl.sum(x32 * x32, axis=1) / D + xn = (x32 * tl.math.rsqrt(var + eps)[:, None]).to(tl.bfloat16) + w = tl.load(w_ptr + offs, mask=offs < D, other=0.0) + h = xn * w[None, :] # bf16 multiply, matching ir.ops.rms_norm + if HAS_GELU: + # Exact (erf) GELU on the bf16-rounded norm output, fp32 math, + # matching F.gelu's opmath on a bf16 tensor. + g32 = h.to(tl.float32) + h = (0.5 * g32 * (1.0 + tldevice.erf(g32 * 0.7071067811865476))).to(tl.bfloat16) + if FOLD: + # Store directly into the next layer's folded layout (a pure + # permutation — replaces the separate fold copy pass). + t = (rows // (FH * FW)) % FT + hh = (rows // FW) % FH + ww = rows % FW + n = rows // (FT * FH * FW) + slot = ((t % TF) * HF + hh % HF) * HF + ww % HF + out_row = ((n * (FT // TF) + t // TF) * (FH // HF) + hh // HF) * ( + FW // HF + ) + ww // HF + base = (out_row * (TF * HF * HF) + slot) * D + tl.store(out_ptr + base[:, None] + offs[None, :], h, mask=mask) + else: + tl.store(out_ptr + rows[:, None] * D + offs[None, :], h, mask=mask) + + +def rmsnorm_gelu( + x: torch.Tensor, # [..., D] bf16 contiguous + weight: torch.Tensor, + eps: float, + gelu: bool = True, + fold: tuple[int, int] | None = None, # (t_fold, hw_fold) of the NEXT fold +) -> torch.Tensor: + """Fused ``gelu(rmsnorm(x))`` (or plain rmsnorm); multiple rows per block + when D is small. With ``fold``, x must be [N, T, H, W, D] and the output + comes back as ``fold_timespace_to_depth(result, *fold)``.""" + D = x.shape[-1] + flat = x.reshape(-1, D) + assert flat.stride(1) == 1 and flat.stride(0) == D + R = flat.shape[0] + if fold is None: + out = torch.empty_like(flat) + ft = fh = fw = tf = hf = 1 + out_shape = x.shape + else: + tf, hf = fold + N, ft, fh, fw, _ = x.shape + out_shape = (N, ft // tf, fh // hf, fw // hf, tf * hf * hf * D) + out = torch.empty(out_shape, dtype=x.dtype, device=x.device) + if R == 0: + return out.reshape(out_shape) + d_p2 = triton.next_power_of_2(D) + block_m = max(1, 4096 // d_p2) + _rmsnorm_gelu_kernel[(triton.cdiv(R, block_m),)]( + flat, + weight, + out, + eps, + R, + D, + D_P2=d_p2, + BLOCK_M=block_m, + HAS_GELU=gelu, + FOLD=fold is not None, + FT=ft, + FH=fh, + FW=fw, + TF=tf, + HF=hf, + num_warps=_get_num_warps_from_block_size(d_p2 * block_m), + ) + return out.reshape(out_shape) diff --git a/vllm/models/inkling/nvidia/ops/norm.py b/vllm/models/inkling/nvidia/ops/norm.py new file mode 100644 index 0000000000..d58a39a06b --- /dev/null +++ b/vllm/models/inkling/nvidia/ops/norm.py @@ -0,0 +1,315 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + +from functools import lru_cache + +import torch + +from vllm.triton_utils import tl, triton + +_MAX_FUSED_SIZE = 65536 + + +def _get_num_warps_from_block_size(block_size: int) -> int: + if block_size >= 32768: + return 32 + if block_size >= 8192: + return 16 + if block_size >= 2048: + return 8 + return 4 + + +def _largest_power_of_2(n: int) -> int: + assert n > 0, f"{n=}" + return 1 << (n.bit_length() - 1) + + +@lru_cache(maxsize=128) +def _get_grid_size_for_mem_bw_kernel(device: torch.device, factor: int = 8) -> int: + num_sms = torch.cuda.get_device_properties(device).multi_processor_count + return _largest_power_of_2(num_sms) * factor + + +@triton.jit +def _rmsnorm_fwd_kernel( + x_ptr, + weight_ptr, + y_ptr, + rstd_ptr, + eps, + x_stride_0, + y_stride_0, + n_cols, + block_size_n: tl.constexpr, +): + pid_m = tl.program_id(0).to(tl.int64) + offs_n = tl.arange(0, block_size_n) + mask_n = offs_n < n_cols + + weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32) + x = tl.load(x_ptr + pid_m * x_stride_0 + offs_n, mask=mask_n, other=0.0).to( + tl.float32 + ) + row_var = tl.sum(x * x, axis=0) / n_cols + rstd = tl.math.rsqrt(row_var + eps) + tl.store(rstd_ptr + pid_m, rstd) + + y = x * rstd * weight + tl.store(y_ptr + pid_m * y_stride_0 + offs_n, y, mask=mask_n) + + +@triton.jit(do_not_specialize=["n_rows"]) +def _rmsnorm_fwd_kernel_block_m( + x_ptr, + weight_ptr, + y_ptr, + rstd_ptr, + eps, + x_stride_0, + y_stride_0, + n_rows, + n_cols, + block_size_m: tl.constexpr, + block_size_n: tl.constexpr, +): + pid_m = tl.program_id(0).to(tl.int64) + offs_n = tl.arange(0, block_size_n) + mask_n = offs_n < n_cols + + num_blocks_m = tl.cdiv(n_rows, block_size_m) + blocks_per_pid = tl.cdiv(num_blocks_m, tl.num_programs(0)) + block_id_start = pid_m * blocks_per_pid + block_id_end = min(block_id_start + blocks_per_pid, num_blocks_m) + + weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32) + + for block_id in range(block_id_start, block_id_end): + offs_m = block_id * block_size_m + tl.arange(0, block_size_m) + mask_m = offs_m < n_rows + mask_mn = mask_m[:, None] & mask_n[None, :] + x = tl.load( + x_ptr + offs_m[:, None] * x_stride_0 + offs_n[None, :], + mask=mask_mn, + other=0.0, + ).to(tl.float32) + row_var = tl.sum(x * x, axis=1) / n_cols + rstd = tl.math.rsqrt(row_var + eps) + tl.store(rstd_ptr + offs_m, rstd, mask=mask_m) + + y = x * rstd[:, None] * weight + tl.store( + y_ptr + offs_m[:, None] * y_stride_0 + offs_n[None, :], + y, + mask=mask_mn, + ) + + +@triton.jit +def _add_rmsnorm_fwd_kernel( + res_ptr, # [T, N] residual (read) + delta_ptr, # [T, N] delta to add (read) + weight_ptr, + y_ptr, # [T, N] normed output + res_out_ptr, # [T, N] updated residual output + eps, + res_stride_0, + delta_stride_0, + y_stride_0, + res_out_stride_0, + n_cols, + block_size_n: tl.constexpr, +): + pid_m = tl.program_id(0).to(tl.int64) + offs_n = tl.arange(0, block_size_n) + mask_n = offs_n < n_cols + + weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32) + r = tl.load(res_ptr + pid_m * res_stride_0 + offs_n, mask=mask_n, other=0.0).to( + tl.float32 + ) + d = tl.load(delta_ptr + pid_m * delta_stride_0 + offs_n, mask=mask_n, other=0.0).to( + tl.float32 + ) + # Round the sum to the residual dtype first (matches the eager + # `residual + delta` then rmsnorm-on-bf16 sequence bit-for-bit). + s = (r + d).to(res_out_ptr.dtype.element_ty) + tl.store(res_out_ptr + pid_m * res_out_stride_0 + offs_n, s, mask=mask_n) + x = s.to(tl.float32) + row_var = tl.sum(x * x, axis=0) / n_cols + rstd = tl.math.rsqrt(row_var + eps) + y = x * rstd * weight + tl.store(y_ptr + pid_m * y_stride_0 + offs_n, y, mask=mask_n) + + +def add_rmsnorm( + residual: torch.Tensor, + delta: torch.Tensor, + weight: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor]: + """Fused ``res = residual + delta; y = rmsnorm(res)``. + + Returns ``(y, res)``; both are fresh tensors (cudagraph-friendly, no + in-place update of the inputs). + """ + assert residual.ndim == 2 and delta.ndim == 2, (residual.shape, delta.shape) + n_rows, n_cols = residual.shape + assert weight.shape[0] == n_cols + y = torch.empty_like(residual) + res_out = torch.empty_like(residual) + if n_rows == 0: + return y, res_out + + block_size_n = triton.next_power_of_2(n_cols) + max_block_size_n = _MAX_FUSED_SIZE // residual.element_size() + if max_block_size_n < block_size_n: + raise RuntimeError(f"Large {n_cols=} is not supported") + num_warps = _get_num_warps_from_block_size(block_size_n) + _add_rmsnorm_fwd_kernel[(n_rows,)]( + residual, + delta, + weight, + y, + res_out, + eps, + residual.stride(0), + delta.stride(0), + y.stride(0), + res_out.stride(0), + n_cols, + block_size_n, + num_warps=num_warps, + ) + return y, res_out + + +@triton.jit +def _embed_rmsnorm_kernel( + ids_ptr, # [T] token ids + table_ptr, # [V, N] embedding table + weight_ptr, # [N] (HAS_NORM only) + chain_weight_ptr, # [N] (HAS_CHAIN only) + out_ptr, # [T, N] rmsnorm(table[ids], weight) + chain_out_ptr, # [T, N] rmsnorm(out, chain_weight) (HAS_CHAIN only) + eps, + table_stride_0, + n_cols, + block_size_n: tl.constexpr, + HAS_NORM: tl.constexpr, + HAS_CHAIN: tl.constexpr, +): + pid_m = tl.program_id(0).to(tl.int64) + offs_n = tl.arange(0, block_size_n) + mask_n = offs_n < n_cols + row = tl.load(ids_ptr + pid_m).to(tl.int64) + x = tl.load(table_ptr + row * table_stride_0 + offs_n, mask=mask_n, other=0.0) + if HAS_NORM: + xf = x.to(tl.float32) + rstd = tl.math.rsqrt(tl.sum(xf * xf, axis=0) / n_cols + eps) + w = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32) + # Round to the output dtype so the chained norm is bit-exact vs the + # unfused pair (which stores bf16 in between). + x = (xf * rstd * w).to(out_ptr.dtype.element_ty) + tl.store(out_ptr + pid_m * n_cols + offs_n, x, mask=mask_n) + if HAS_CHAIN: + xf = x.to(tl.float32) + rstd = tl.math.rsqrt(tl.sum(xf * xf, axis=0) / n_cols + eps) + w = tl.load(chain_weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32) + tl.store( + chain_out_ptr + pid_m * n_cols + offs_n, + (xf * rstd * w).to(chain_out_ptr.dtype.element_ty), + mask=mask_n, + ) + + +def embed_rmsnorm( + input_ids: torch.Tensor, + embed_table: torch.Tensor, + weight: torch.Tensor | None, + eps: float, + chain_weight: torch.Tensor | None = None, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + """Fused ``rmsnorm(embed_table[input_ids], weight)`` row gather + norm. + + Requires the full vocab on-rank (replicated or tp_size == 1). + ``weight=None`` skips the norm (``use_embed_norm=False``), leaving a pure + embedding-table gather. + ``chain_weight`` additionally emits ``rmsnorm(out, chain_weight)`` (the + first decoder layer's pre-attention norm) as a second output, still one + launch. Bit-exact vs the unfused module sequence.""" + ids = input_ids.view(-1) + (T,) = ids.shape + n = embed_table.shape[1] + out = torch.empty( + (*input_ids.shape, n), dtype=embed_table.dtype, device=embed_table.device + ) + chain_out = torch.empty_like(out) if chain_weight is not None else None + if T > 0: + block_size_n = triton.next_power_of_2(n) + _embed_rmsnorm_kernel[(T,)]( + ids, + embed_table, + weight if weight is not None else embed_table, + chain_weight if chain_weight is not None else embed_table, + out, + chain_out if chain_out is not None else out, + eps, + embed_table.stride(0), + n, + block_size_n, + HAS_NORM=weight is not None, + HAS_CHAIN=chain_weight is not None, + num_warps=_get_num_warps_from_block_size(block_size_n), + ) + if chain_out is not None: + return out, chain_out + return out + + +def rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: + assert x.ndim == 2, f"{x.shape=}" + assert weight.ndim == 1, f"{weight.shape=}" + n_rows, n_cols = x.shape + assert weight.shape[0] == n_cols, f"{weight.shape=} {x.shape=}" + y = torch.empty_like(x) + rstd = torch.empty((n_rows,), dtype=torch.float32, device=x.device) + + block_size_n = triton.next_power_of_2(n_cols) + max_block_size_n = _MAX_FUSED_SIZE // x.element_size() + if max_block_size_n < block_size_n: + raise RuntimeError(f"Large {n_cols=} is not supported") + block_size_m = max(1, 4096 // block_size_n) + num_warps = _get_num_warps_from_block_size(block_size_n) + + if block_size_m == 1: + _rmsnorm_fwd_kernel[(n_rows,)]( + x, + weight, + y, + rstd, + eps, + x.stride(0), + y.stride(0), + n_cols, + block_size_n, + num_warps=num_warps, + ) + else: + grid_size = _get_grid_size_for_mem_bw_kernel(x.device) + _rmsnorm_fwd_kernel_block_m[(grid_size,)]( + x, + weight, + y, + rstd, + eps, + x.stride(0), + y.stride(0), + n_rows, + n_cols, + block_size_m, + block_size_n, + num_warps=num_warps, + ) + return y diff --git a/vllm/models/inkling/nvidia/ops/qkvr_prep.py b/vllm/models/inkling/nvidia/ops/qkvr_prep.py new file mode 100644 index 0000000000..a910e72170 --- /dev/null +++ b/vllm/models/inkling/nvidia/ops/qkvr_prep.py @@ -0,0 +1,918 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import aux_stream + +LOW_BLOCK_M = 32 +LOW_BLOCK_N = 64 +LOW_NUM_WARPS = 4 +THROUGHPUT_BLOCK_M = 32 +THROUGHPUT_BLOCK_N = 128 +THROUGHPUT_GROUP_M = 2 +THROUGHPUT_NUM_WARPS = 4 +SMALL_TOKEN_THRESHOLD = 128 +SMALL_NUM_WARPS = 2 +Q_BLOCK_ROWS = 8 +Q_NUM_WARPS = 2 +KV_BLOCK_ROWS = 4 +KV_NUM_WARPS = 2 + + +@triton.jit(do_not_specialize=["rows"]) +def _rel_proj_low_latency_kernel( + qkvr_ptr, + rel_proj_ptr, + rel_out_ptr, + log_scaling_ptr, + rows, + stride_x_t, + R_OFFSET: tl.constexpr, + NUM_Q_HEADS: tl.constexpr, + REL_EXTENT: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + APPLY_LOG_SCALING: tl.constexpr, +): + row = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M) + col = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) + inner = tl.arange(0, 16) + token = row // NUM_Q_HEADS + head = row % NUM_Q_HEADS + relative = tl.load( + qkvr_ptr + + token[:, None] * stride_x_t + + R_OFFSET + + head[:, None] * 16 + + inner[None, :], + mask=row[:, None] < rows, + other=0.0, + ) + projection = tl.load( + rel_proj_ptr + inner[:, None] * REL_EXTENT + col[None, :], + mask=col[None, :] < REL_EXTENT, + other=0.0, + ) + values = tl.dot(relative, projection, out_dtype=tl.float32).to( + rel_out_ptr.dtype.element_ty + ) + values = values.to(tl.float32) + if APPLY_LOG_SCALING: + values *= tl.load(log_scaling_ptr + token, mask=row < rows, other=1.0)[:, None] + tl.store( + rel_out_ptr + row[:, None] * REL_EXTENT + col[None, :], + values.to(rel_out_ptr.dtype.element_ty), + mask=(row[:, None] < rows) & (col[None, :] < REL_EXTENT), + ) + + +@triton.jit(do_not_specialize=["rows"]) +def _rel_proj_throughput_kernel( + qkvr_ptr, + rel_proj_ptr, + rel_out_ptr, + log_scaling_ptr, + rows, + stride_x_t, + R_OFFSET: tl.constexpr, + NUM_Q_HEADS: tl.constexpr, + REL_EXTENT: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + GROUP_M: tl.constexpr, + APPLY_LOG_SCALING: tl.constexpr, +): + row_group = tl.program_id(0) + col = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) + inner = tl.arange(0, 16) + projection = tl.load( + rel_proj_ptr + inner[:, None] * REL_EXTENT + col[None, :], + mask=col[None, :] < REL_EXTENT, + other=0.0, + ) + row_offsets = tl.arange(0, BLOCK_M) + for group_offset in tl.static_range(GROUP_M): + row = (row_group * GROUP_M + group_offset) * BLOCK_M + row_offsets + token = row // NUM_Q_HEADS + head = row % NUM_Q_HEADS + relative = tl.load( + qkvr_ptr + + token[:, None] * stride_x_t + + R_OFFSET + + head[:, None] * 16 + + inner[None, :], + mask=row[:, None] < rows, + other=0.0, + ) + values = tl.dot(relative, projection, out_dtype=tl.float32).to( + rel_out_ptr.dtype.element_ty + ) + values = values.to(tl.float32) + if APPLY_LOG_SCALING: + values *= tl.load(log_scaling_ptr + token, mask=row < rows, other=1.0)[ + :, None + ] + tl.store( + rel_out_ptr + row[:, None] * REL_EXTENT + col[None, :], + values.to(rel_out_ptr.dtype.element_ty), + mask=(row[:, None] < rows) & (col[None, :] < REL_EXTENT), + ) + + +def use_rel_proj_throughput(rows: int, rel_extent: int) -> bool: + min_rows = 8192 if rel_extent == 512 else 2048 + return rows >= min_rows + + +def qkvr_rel_proj( + qkvr: torch.Tensor, + rel_proj: torch.Tensor, + rel_out: torch.Tensor, + log_scaling: torch.Tensor | None, + *, + num_q_heads: int, + num_kv_heads: int, + head_dim: int, + d_rel: int, +) -> None: + rows = qkvr.shape[0] * num_q_heads + rel_extent = rel_proj.shape[1] + assert d_rel == 16 and rel_proj.shape[0] == 16 + r_offset = num_q_heads * head_dim + 2 * num_kv_heads * head_dim + log_scaling_ptr = log_scaling if log_scaling is not None else qkvr + common = dict( + R_OFFSET=r_offset, + NUM_Q_HEADS=num_q_heads, + REL_EXTENT=rel_extent, + APPLY_LOG_SCALING=log_scaling is not None, + ) + + if use_rel_proj_throughput(rows, rel_extent): + grid = ( + triton.cdiv(rows, THROUGHPUT_BLOCK_M * THROUGHPUT_GROUP_M), + triton.cdiv(rel_extent, THROUGHPUT_BLOCK_N), + ) + _rel_proj_throughput_kernel[grid]( + qkvr, + rel_proj, + rel_out, + log_scaling_ptr, + rows, + qkvr.stride(0), + BLOCK_M=THROUGHPUT_BLOCK_M, + BLOCK_N=THROUGHPUT_BLOCK_N, + GROUP_M=THROUGHPUT_GROUP_M, + num_warps=THROUGHPUT_NUM_WARPS, + **common, + ) + return + + grid = ( + triton.cdiv(rows, LOW_BLOCK_M), + triton.cdiv(rel_extent, LOW_BLOCK_N), + ) + _rel_proj_low_latency_kernel[grid]( + qkvr, + rel_proj, + rel_out, + log_scaling_ptr, + rows, + qkvr.stride(0), + BLOCK_M=LOW_BLOCK_M, + BLOCK_N=LOW_BLOCK_N, + num_warps=LOW_NUM_WARPS, + **common, + ) + + +@triton.jit(do_not_specialize=["tokens", "stride_block_table_req", "max_blocks"]) +def _qkvr_qkv_kernel( + qkvr_ptr, + q_norm_weight_ptr, + q_out_ptr, + rel_proj_ptr, + rel_out_ptr, + k_weight_ptr, + v_weight_ptr, + k_norm_weight_ptr, + conv_cache_ptr, + key_cache_ptr, + value_cache_ptr, + positions_ptr, + seq_idx_ptr, + conv_slot_mapping_ptr, + conv_block_table_ptr, + query_start_ptr, + attention_slot_mapping_ptr, + log_scaling_ptr, + tokens, + eps, + stride_x_t, + stride_cc_block, + stride_cc_head, + stride_cc_token, + stride_cc_dim, + stride_kc_block, + stride_kc_token, + stride_kc_head, + stride_vc_block, + stride_vc_token, + stride_vc_head, + stride_block_table_req, + max_blocks, + conv_block_size, + attention_page_size, + Q_WIDTH: tl.constexpr, + KV_WIDTH: tl.constexpr, + NUM_Q_HEADS: tl.constexpr, + NUM_KV_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + WINDOW_SIZE: tl.constexpr, + OFF_K: tl.constexpr, + OFF_V: tl.constexpr, + APPLY_LOG_SCALING: tl.constexpr, + D_REL: tl.constexpr, + REL_EXTENT: tl.constexpr, + REL_EXTENT_PADDED: tl.constexpr, +): + block = tl.program_id(0) + num_q_rows = tokens * NUM_Q_HEADS + dims = tl.arange(0, HEAD_DIM) + + if block < num_q_rows: + row = block + token = row // NUM_Q_HEADS + head = row % NUM_Q_HEADS + values = tl.load( + qkvr_ptr + token * stride_x_t + head * HEAD_DIM + dims, + ).to(tl.float32) + weight = tl.load(q_norm_weight_ptr + dims).to(tl.float32) + rstd = tl.rsqrt(tl.sum(values * values, axis=0) / HEAD_DIM + eps) + normalized = values * rstd * weight + if APPLY_LOG_SCALING: + normalized = normalized.to(q_out_ptr.dtype.element_ty).to(tl.float32) + normalized *= tl.load(log_scaling_ptr + token) + tl.store( + q_out_ptr + row * HEAD_DIM + dims, + normalized.to(q_out_ptr.dtype.element_ty), + ) + rel_cols = tl.arange(0, REL_EXTENT_PADDED) + rel_mask = rel_cols < REL_EXTENT + projected = tl.zeros([REL_EXTENT_PADDED], dtype=tl.float32) + rel_offset = Q_WIDTH + 2 * KV_WIDTH + head * D_REL + for rel_dim in tl.static_range(D_REL): + rel_value = tl.load( + qkvr_ptr + token * stride_x_t + rel_offset + rel_dim + ).to(tl.float32) + proj = tl.load( + rel_proj_ptr + rel_dim * REL_EXTENT + rel_cols, + mask=rel_mask, + other=0.0, + ).to(tl.float32) + projected += rel_value * proj + projected = projected.to(rel_out_ptr.dtype.element_ty).to(tl.float32) + if APPLY_LOG_SCALING: + projected *= tl.load(log_scaling_ptr + token) + tl.store( + rel_out_ptr + row * REL_EXTENT + rel_cols, + projected.to(rel_out_ptr.dtype.element_ty), + mask=rel_mask, + ) + else: + row = block - num_q_rows + if row < tokens * NUM_KV_HEADS: + token = row // NUM_KV_HEADS + head = row % NUM_KV_HEADS + position = tl.load(positions_ptr + token) + request = tl.load(seq_idx_ptr + token) + conv_slot = tl.load(conv_slot_mapping_ptr + token) + query_start = tl.load(query_start_ptr + token) + attention_slot = tl.load(attention_slot_mapping_ptr + token) + valid = conv_slot >= 0 + + k_col = Q_WIDTH + head * HEAD_DIM + v_col = Q_WIDTH + KV_WIDTH + head * HEAD_DIM + k_value = tl.load(qkvr_ptr + token * stride_x_t + k_col + dims) + v_value = tl.load(qkvr_ptr + token * stride_x_t + v_col + dims) + + safe_slot = tl.maximum(conv_slot, 0) + cache_base = ( + conv_cache_ptr + + (safe_slot // conv_block_size) * stride_cc_block + + head * stride_cc_head + + (safe_slot % conv_block_size) * stride_cc_token + ) + tl.store( + cache_base + (OFF_K + dims) * stride_cc_dim, + k_value, + mask=valid, + ) + tl.store( + cache_base + (OFF_V + dims) * stride_cc_dim, + v_value, + mask=valid, + ) + + acc_k = tl.zeros([HEAD_DIM], dtype=tl.float32) + acc_v = tl.zeros([HEAD_DIM], dtype=tl.float32) + for tap in tl.static_range(WINDOW_SIZE): + source_position = position - (WINDOW_SIZE - 1) + tap + source_row = token - (WINDOW_SIZE - 1) + tap + in_window = valid & (source_position >= 0) + intra = in_window & (source_row >= query_start) + cached = in_window & (source_row < query_start) + safe_row = tl.maximum(source_row, 0) + source_k = tl.load( + qkvr_ptr + safe_row * stride_x_t + k_col + dims, + mask=intra, + other=0.0, + ).to(tl.float32) + source_v = tl.load( + qkvr_ptr + safe_row * stride_x_t + v_col + dims, + mask=intra, + other=0.0, + ).to(tl.float32) + safe_position = tl.maximum(source_position, 0) + logical_block = tl.minimum( + safe_position // conv_block_size, max_blocks - 1 + ) + physical_block = tl.load( + conv_block_table_ptr + + request * stride_block_table_req + + logical_block, + mask=cached, + other=0, + ).to(tl.int64) + tap_base = ( + conv_cache_ptr + + physical_block * stride_cc_block + + head * stride_cc_head + + (safe_position % conv_block_size) * stride_cc_token + ) + source_k += tl.load( + tap_base + (OFF_K + dims) * stride_cc_dim, + mask=cached, + other=0.0, + ).to(tl.float32) + source_v += tl.load( + tap_base + (OFF_V + dims) * stride_cc_dim, + mask=cached, + other=0.0, + ).to(tl.float32) + k_weight = tl.load( + k_weight_ptr + (head * HEAD_DIM + dims) * WINDOW_SIZE + tap + ).to(tl.float32) + v_weight = tl.load( + v_weight_ptr + (head * HEAD_DIM + dims) * WINDOW_SIZE + tap + ).to(tl.float32) + acc_k += source_k * k_weight + acc_v += source_v * v_weight + + k_rounded = (acc_k + k_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty) + v_rounded = (acc_v + v_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty) + k_float = k_rounded.to(tl.float32) + k_norm_weight = tl.load(k_norm_weight_ptr + dims).to(tl.float32) + rstd = tl.rsqrt(tl.sum(k_float * k_float, axis=0) / HEAD_DIM + eps) + k_normalized = (k_float * rstd * k_norm_weight).to( + qkvr_ptr.dtype.element_ty + ) + + safe_attention_slot = tl.maximum(attention_slot, 0) + attention_block = safe_attention_slot // attention_page_size + attention_offset = safe_attention_slot % attention_page_size + tl.store( + key_cache_ptr + + attention_block * stride_kc_block + + attention_offset * stride_kc_token + + head * stride_kc_head + + dims, + k_normalized, + mask=attention_slot >= 0, + ) + tl.store( + value_cache_ptr + + attention_block * stride_vc_block + + attention_offset * stride_vc_token + + head * stride_vc_head + + dims, + v_rounded, + mask=attention_slot >= 0, + ) + + +@triton.jit(do_not_specialize=["num_rows"]) +def _q_kernel( + qkvr_ptr, + q_norm_weight_ptr, + q_out_ptr, + log_scaling_ptr, + num_rows, + stride_x_t, + eps, + NUM_Q_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_ROWS: tl.constexpr, + APPLY_LOG_SCALING: tl.constexpr, +): + rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS) + dims = tl.arange(0, HEAD_DIM) + row_mask = rows < num_rows + tokens = rows // NUM_Q_HEADS + heads = rows % NUM_Q_HEADS + values = tl.load( + qkvr_ptr + + tokens[:, None] * stride_x_t + + heads[:, None] * HEAD_DIM + + dims[None, :], + mask=row_mask[:, None], + other=0.0, + ).to(tl.float32) + weight = tl.load(q_norm_weight_ptr + dims).to(tl.float32) + rstd = tl.rsqrt(tl.sum(values * values, axis=1) / HEAD_DIM + eps) + normalized = values * rstd[:, None] * weight[None, :] + if APPLY_LOG_SCALING: + normalized = normalized.to(q_out_ptr.dtype.element_ty).to(tl.float32) + tau = tl.load(log_scaling_ptr + tokens, mask=row_mask, other=1.0) + normalized *= tau[:, None] + tl.store( + q_out_ptr + rows[:, None] * HEAD_DIM + dims[None, :], + normalized.to(q_out_ptr.dtype.element_ty), + mask=row_mask[:, None], + ) + + +@triton.jit(do_not_specialize=["tokens", "stride_block_table_req", "max_blocks"]) +def _kv_kernel( + qkvr_ptr, + k_weight_ptr, + v_weight_ptr, + k_norm_weight_ptr, + conv_cache_ptr, + key_cache_ptr, + value_cache_ptr, + positions_ptr, + seq_idx_ptr, + conv_slot_mapping_ptr, + conv_block_table_ptr, + query_start_ptr, + attention_slot_mapping_ptr, + tokens, + eps, + stride_x_t, + stride_cc_block, + stride_cc_head, + stride_cc_token, + stride_cc_dim, + stride_kc_block, + stride_kc_token, + stride_kc_head, + stride_vc_block, + stride_vc_token, + stride_vc_head, + stride_block_table_req, + max_blocks, + conv_block_size, + attention_page_size, + Q_WIDTH: tl.constexpr, + KV_WIDTH: tl.constexpr, + NUM_KV_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + WINDOW_SIZE: tl.constexpr, + OFF_K: tl.constexpr, + OFF_V: tl.constexpr, + BLOCK_ROWS: tl.constexpr, +): + token = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS) + dims = tl.arange(0, HEAD_DIM) + row_mask = token < tokens + head_id = tl.program_id(1) + head = tl.full([BLOCK_ROWS], head_id, tl.int64) + position = tl.load(positions_ptr + token, mask=row_mask, other=0) + request = tl.load(seq_idx_ptr + token, mask=row_mask, other=0) + conv_slot = tl.load(conv_slot_mapping_ptr + token, mask=row_mask, other=-1) + query_start = tl.load(query_start_ptr + token, mask=row_mask, other=0) + attention_slot = tl.load( + attention_slot_mapping_ptr + token, mask=row_mask, other=-1 + ) + valid = row_mask & (conv_slot >= 0) + + k_col = Q_WIDTH + head * HEAD_DIM + v_col = Q_WIDTH + KV_WIDTH + head * HEAD_DIM + k_value = tl.load( + qkvr_ptr + token[:, None] * stride_x_t + k_col[:, None] + dims[None, :], + mask=row_mask[:, None], + other=0.0, + ) + v_value = tl.load( + qkvr_ptr + token[:, None] * stride_x_t + v_col[:, None] + dims[None, :], + mask=row_mask[:, None], + other=0.0, + ) + safe_slot = tl.maximum(conv_slot, 0) + cache_base = ( + conv_cache_ptr + + (safe_slot // conv_block_size) * stride_cc_block + + head * stride_cc_head + + (safe_slot % conv_block_size) * stride_cc_token + ) + tl.store( + cache_base[:, None] + (OFF_K + dims[None, :]) * stride_cc_dim, + k_value, + mask=valid[:, None], + ) + tl.store( + cache_base[:, None] + (OFF_V + dims[None, :]) * stride_cc_dim, + v_value, + mask=valid[:, None], + ) + + acc_k = tl.zeros([BLOCK_ROWS, HEAD_DIM], dtype=tl.float32) + acc_v = tl.zeros([BLOCK_ROWS, HEAD_DIM], dtype=tl.float32) + for tap in tl.static_range(WINDOW_SIZE): + source_position = position - (WINDOW_SIZE - 1) + tap + source_row = token - (WINDOW_SIZE - 1) + tap + in_window = valid & (source_position >= 0) + intra = in_window & (source_row >= query_start) + cached = in_window & (source_row < query_start) + safe_row = tl.maximum(source_row, 0) + source_k = tl.load( + qkvr_ptr + safe_row[:, None] * stride_x_t + k_col[:, None] + dims[None, :], + mask=intra[:, None], + other=0.0, + ).to(tl.float32) + source_v = tl.load( + qkvr_ptr + safe_row[:, None] * stride_x_t + v_col[:, None] + dims[None, :], + mask=intra[:, None], + other=0.0, + ).to(tl.float32) + safe_position = tl.maximum(source_position, 0) + logical_block = tl.minimum(safe_position // conv_block_size, max_blocks - 1) + physical_block = tl.load( + conv_block_table_ptr + request * stride_block_table_req + logical_block, + mask=cached, + other=0, + ).to(tl.int64) + tap_base = ( + conv_cache_ptr + + physical_block * stride_cc_block + + head * stride_cc_head + + (safe_position % conv_block_size) * stride_cc_token + ) + source_k += tl.load( + tap_base[:, None] + (OFF_K + dims[None, :]) * stride_cc_dim, + mask=cached[:, None], + other=0.0, + ).to(tl.float32) + source_v += tl.load( + tap_base[:, None] + (OFF_V + dims[None, :]) * stride_cc_dim, + mask=cached[:, None], + other=0.0, + ).to(tl.float32) + k_weight = tl.load( + k_weight_ptr + (head_id * HEAD_DIM + dims) * WINDOW_SIZE + tap + ).to(tl.float32) + v_weight = tl.load( + v_weight_ptr + (head_id * HEAD_DIM + dims) * WINDOW_SIZE + tap + ).to(tl.float32) + acc_k += source_k * k_weight[None, :] + acc_v += source_v * v_weight[None, :] + + k_rounded = (acc_k + k_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty) + v_rounded = (acc_v + v_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty) + k_float = k_rounded.to(tl.float32) + k_norm_weight = tl.load(k_norm_weight_ptr + dims).to(tl.float32) + rstd = tl.rsqrt(tl.sum(k_float * k_float, axis=1) / HEAD_DIM + eps) + k_normalized = (k_float * rstd[:, None] * k_norm_weight[None, :]).to( + qkvr_ptr.dtype.element_ty + ) + + safe_attention_slot = tl.maximum(attention_slot, 0) + attention_block = safe_attention_slot // attention_page_size + attention_offset = safe_attention_slot % attention_page_size + attention_mask = row_mask & (attention_slot >= 0) + tl.store( + key_cache_ptr + + attention_block[:, None] * stride_kc_block + + attention_offset[:, None] * stride_kc_token + + head[:, None] * stride_kc_head + + dims[None, :], + k_normalized, + mask=attention_mask[:, None], + ) + tl.store( + value_cache_ptr + + attention_block[:, None] * stride_vc_block + + attention_offset[:, None] * stride_vc_token + + head[:, None] * stride_vc_head + + dims[None, :], + v_rounded, + mask=attention_mask[:, None], + ) + + +def _run_tiled_q( + qkvr: torch.Tensor, + q_norm_weight: torch.Tensor, + q_out: torch.Tensor, + positions: torch.Tensor, + *, + eps: float, + num_q_heads: int, + head_dim: int, + log_scaling: torch.Tensor | None, +) -> None: + num_rows = qkvr.shape[0] * num_q_heads + _q_kernel[(triton.cdiv(num_rows, Q_BLOCK_ROWS),)]( + qkvr, + q_norm_weight, + q_out, + log_scaling if log_scaling is not None else positions, + num_rows, + qkvr.stride(0), + eps, + NUM_Q_HEADS=num_q_heads, + HEAD_DIM=head_dim, + BLOCK_ROWS=Q_BLOCK_ROWS, + APPLY_LOG_SCALING=log_scaling is not None, + num_warps=Q_NUM_WARPS, + ) + + +def _run_tiled_kv( + qkvr: torch.Tensor, + k_weight: torch.Tensor, + v_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + conv_cache: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + positions: torch.Tensor, + seq_idx: torch.Tensor, + conv_slot_mapping: torch.Tensor, + conv_block_table: torch.Tensor, + query_start: torch.Tensor, + attention_slot_mapping: torch.Tensor, + *, + eps: float, + num_q_heads: int, + num_kv_heads: int, + head_dim: int, + off_k: int, + off_v: int, + conv_block_size: int, +) -> None: + tokens = qkvr.shape[0] + _kv_kernel[(triton.cdiv(tokens, KV_BLOCK_ROWS), num_kv_heads)]( + qkvr, + k_weight, + v_weight, + k_norm_weight, + conv_cache, + key_cache, + value_cache, + positions, + seq_idx, + conv_slot_mapping, + conv_block_table, + query_start, + attention_slot_mapping, + tokens, + eps, + qkvr.stride(0), + conv_cache.stride(0), + conv_cache.stride(1), + conv_cache.stride(2), + conv_cache.stride(3), + key_cache.stride(0), + key_cache.stride(1), + key_cache.stride(2), + value_cache.stride(0), + value_cache.stride(1), + value_cache.stride(2), + conv_block_table.stride(0), + conv_block_table.shape[1], + conv_block_size, + key_cache.shape[1], + Q_WIDTH=num_q_heads * head_dim, + KV_WIDTH=num_kv_heads * head_dim, + NUM_KV_HEADS=num_kv_heads, + HEAD_DIM=head_dim, + WINDOW_SIZE=k_weight.shape[1], + OFF_K=off_k, + OFF_V=off_v, + BLOCK_ROWS=KV_BLOCK_ROWS, + num_warps=KV_NUM_WARPS, + ) + + +def _run_fused_small( + qkvr: torch.Tensor, + q_norm_weight: torch.Tensor, + q_out: torch.Tensor, + rel_proj: torch.Tensor, + rel_out: torch.Tensor, + k_weight: torch.Tensor, + v_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + conv_cache: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + positions: torch.Tensor, + seq_idx: torch.Tensor, + conv_slot_mapping: torch.Tensor, + conv_block_table: torch.Tensor, + query_start: torch.Tensor, + attention_slot_mapping: torch.Tensor, + *, + eps: float, + num_q_heads: int, + num_kv_heads: int, + head_dim: int, + off_k: int, + off_v: int, + conv_block_size: int, + log_scaling: torch.Tensor | None, +) -> None: + tokens = qkvr.shape[0] + + num_q_rows = tokens * num_q_heads + grid = (num_q_rows + tokens * num_kv_heads,) + _qkvr_qkv_kernel[grid]( + qkvr, + q_norm_weight, + q_out, + rel_proj, + rel_out, + k_weight, + v_weight, + k_norm_weight, + conv_cache, + key_cache, + value_cache, + positions, + seq_idx, + conv_slot_mapping, + conv_block_table, + query_start, + attention_slot_mapping, + log_scaling if log_scaling is not None else positions, + tokens, + eps, + qkvr.stride(0), + conv_cache.stride(0), + conv_cache.stride(1), + conv_cache.stride(2), + conv_cache.stride(3), + key_cache.stride(0), + key_cache.stride(1), + key_cache.stride(2), + value_cache.stride(0), + value_cache.stride(1), + value_cache.stride(2), + conv_block_table.stride(0), + conv_block_table.shape[1], + conv_block_size, + key_cache.shape[1], + Q_WIDTH=num_q_heads * head_dim, + KV_WIDTH=num_kv_heads * head_dim, + NUM_Q_HEADS=num_q_heads, + NUM_KV_HEADS=num_kv_heads, + HEAD_DIM=head_dim, + WINDOW_SIZE=k_weight.shape[1], + OFF_K=off_k, + OFF_V=off_v, + APPLY_LOG_SCALING=log_scaling is not None, + D_REL=16, + REL_EXTENT=rel_proj.shape[1], + REL_EXTENT_PADDED=triton.next_power_of_2(rel_proj.shape[1]), + num_warps=SMALL_NUM_WARPS, + ) + + +def fused_qkvr_prep( + qkvr: torch.Tensor, + k_weight: torch.Tensor, + v_weight: torch.Tensor, + q_norm_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + rel_proj: torch.Tensor, + eps: float, + num_q_heads: int, + num_kv_heads: int, + head_dim: int, + d_rel: int, + conv_cache: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + positions: torch.Tensor, + conv_block_table: torch.Tensor, + seq_idx: torch.Tensor, + conv_slot_mapping: torch.Tensor, + query_start: torch.Tensor, + attention_slot_mapping: torch.Tensor, + off_k: int, + off_v: int, + conv_block_size: int, + log_scaling: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + assert d_rel == 16 and rel_proj.shape[0] == 16 + assert head_dim == 128 + assert qkvr.is_contiguous() + assert k_weight.stride() == (k_weight.shape[1], 1) + assert v_weight.stride() == (v_weight.shape[1], 1) + assert rel_proj.stride() == (rel_proj.shape[1], 1) + assert conv_cache.stride(3) == 1 + assert key_cache.stride(3) == 1 and value_cache.stride(3) == 1 + tokens = qkvr.shape[0] + q_out = torch.empty( + (tokens, num_q_heads * head_dim), dtype=qkvr.dtype, device=qkvr.device + ) + rel_out = torch.empty( + (tokens, num_q_heads, rel_proj.shape[1]), + dtype=qkvr.dtype, + device=qkvr.device, + ) + if tokens == 0: + return q_out, rel_out + + if tokens < SMALL_TOKEN_THRESHOLD: + _run_fused_small( + qkvr, + q_norm_weight, + q_out, + rel_proj, + rel_out, + k_weight, + v_weight, + k_norm_weight, + conv_cache, + key_cache, + value_cache, + positions, + seq_idx, + conv_slot_mapping, + conv_block_table, + query_start, + attention_slot_mapping, + eps=eps, + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + off_k=off_k, + off_v=off_v, + conv_block_size=conv_block_size, + log_scaling=log_scaling, + ) + return q_out, rel_out + + kv_stream = aux_stream() + assert kv_stream is not None + current_stream = torch.cuda.current_stream() + kv_stream.wait_stream(current_stream) + with torch.cuda.stream(kv_stream): + _run_tiled_kv( + qkvr, + k_weight, + v_weight, + k_norm_weight, + conv_cache, + key_cache, + value_cache, + positions, + seq_idx, + conv_slot_mapping, + conv_block_table, + query_start, + attention_slot_mapping, + eps=eps, + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + off_k=off_k, + off_v=off_v, + conv_block_size=conv_block_size, + ) + _run_tiled_q( + qkvr, + q_norm_weight, + q_out, + positions, + eps=eps, + num_q_heads=num_q_heads, + head_dim=head_dim, + log_scaling=log_scaling, + ) + qkvr_rel_proj( + qkvr, + rel_proj, + rel_out, + log_scaling, + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + d_rel=d_rel, + ) + current_stream.wait_stream(kv_stream) + return q_out, rel_out diff --git a/vllm/models/inkling/nvidia/ops/sconv.py b/vllm/models/inkling/nvidia/ops/sconv.py new file mode 100644 index 0000000000..f21ff6e04c --- /dev/null +++ b/vllm/models/inkling/nvidia/ops/sconv.py @@ -0,0 +1,290 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling short-convolution kernels backed by a paged sliding-window state cache. + +Each layer's 4 conv streams (K, V, attn-output, mlp-output) share one paged KV +cache ``[num_blocks, H, N, D]`` (head-major; see ``sconv_swa_attn.py``). A stream +occupies the contiguous D-sub-range ``[off_s, off_s + ws)`` across all ``H`` +heads, so its flat per-token width is ``H * ws`` and that is the conv channel +dim. The cache stores the conv *input* at every absolute position. + +``fused_sconv`` is the single-launch path used by the model: per token it writes +the current input to its slot and convolves the ``W`` taps ending at its +absolute position. A tap landing inside the current forward is read from the +immutable input ``x`` (row ``src - pos + pid_t``); only pre-forward taps are read +from the paged cache (window position ``src`` -> physical block via +``block_table[req, src // N]``). The just-written slot is never read back this +step, so there is no write/read hazard within or across programs -- which is +why this needs no decode-vs-prefill split and is valid for prefill / decode / +spec alike. + +All kernels address the cache purely by ``(slot, absolute_position)`` and +allocate nothing inside the captured region; their grids depend only on the +token count (``fused_sconv`` on a fixed token/channel tiling), so the same +decode path replays correctly under a full CUDA graph without any +data-dependent shape or branch. +""" + +from __future__ import annotations + +import torch + +from vllm.triton_utils import tl, triton + + +@triton.jit +def _fused_sconv_kernel( + x_ptr, # [T, H*WS] head-major current-token inputs (also residual) + cache_ptr, # [num_blocks, H, N, D] paged (page-strided view) + weight_ptr, # [H*WS, W] + out_ptr, # [T, H*WS] + pos_ptr, # [T] int64 absolute position per token + seq_idx_ptr, # [T] int32 token -> batch request + slot_ptr, # [T] int64 flat slot (block*N + blk_off); < 0 => PAD + block_table_ptr, # [num_reqs, max_blocks] int32 block_table + qstart_ptr, # [T] int32 first x-row of the token's request + T, # num tokens + stride_x_t, + stride_c_blk, + stride_c_h, + stride_c_n, + stride_c_d, + stride_w_d, + stride_w_w, + stride_bt_r, + MAX_BLOCKS, + N, # block_size + W: tl.constexpr, + USE_SILU: tl.constexpr, + USE_RESIDUAL: tl.constexpr, + OFF_S: tl.constexpr, + WS: tl.constexpr, + H: tl.constexpr, + BT: tl.constexpr, + BLOCK_C: tl.constexpr, +): + # Each program owns a [BT tokens, BLOCK_C channels] tile. The flat channel + # index packs all H heads head-major (head = c // WS, in-stream offset = + # c % WS), so one program spans heads -- no per-head launch and no + # next_power_of_2(WS) lane waste. + pid_t = tl.program_id(0) + pid_c = tl.program_id(1) + toff = pid_t * BT + tl.arange(0, BT) # [BT] token rows + coff = pid_c * BLOCK_C + tl.arange(0, BLOCK_C) # [BLOCK_C] flat channels + C = H * WS + t_mask = toff < T + c_mask = coff < C + head = tl.minimum(coff // WS, H - 1) # clamp keeps masked lanes in-buffer + cd = OFF_S + coff % WS # cache D-index of the channel's stream slot + + slot = tl.load(slot_ptr + toff, mask=t_mask, other=-1) # [BT] + valid = slot >= 0 + pos = tl.load(pos_ptr + toff, mask=t_mask, other=0) + req = tl.load(seq_idx_ptr + toff, mask=t_mask, other=0) + qstart = tl.load(qstart_ptr + toff, mask=t_mask, other=0) + + tc_mask = t_mask[:, None] & c_mask[None, :] + + # 1) Insert each token's input into its paged slot (skip PAD rows). + xv = tl.load(x_ptr + toff[:, None] * stride_x_t + coff[None, :], mask=tc_mask) + safe_slot = tl.maximum(slot, 0) + dst = ( + cache_ptr + + (safe_slot // N)[:, None] * stride_c_blk + + head[None, :] * stride_c_h + + (safe_slot % N)[:, None] * stride_c_n + + cd[None, :] * stride_c_d + ) + tl.store(dst, xv, mask=tc_mask & valid[:, None]) + + # 2) Convolve the W taps ending at each token's `pos`. Each tap is read from + # `x` if it falls inside this forward (row >= the request's first row), else + # from the paged cache. Exactly one source is unmasked per tap, so we sum both. + acc = tl.zeros([BT, BLOCK_C], dtype=tl.float32) + for iw in tl.static_range(W): + src = pos - (W - 1) + iw # [BT] absolute window position + row = toff - (W - 1) + iw # [BT] x-row of `src` (== src - pos + token) + in_win = valid & (src >= 0) + intra = in_win & (row >= qstart) + cached = in_win & (row < qstart) + # intra-forward tap: read the immutable input x (never the slot we just + # wrote), so there is no write/read hazard. + safe_row = tl.maximum(row, 0) + xt = tl.load( + x_ptr + safe_row[:, None] * stride_x_t + coff[None, :], + mask=c_mask[None, :] & intra[:, None], + other=0.0, + ).to(tl.float32) + # pre-forward tap: read from the paged cache via the block table. Clamp + # addressing terms; the load is masked off when out of window. + safe_src = tl.maximum(src, 0) + safe_lblk = tl.minimum(safe_src // N, MAX_BLOCKS - 1) + blk = tl.load( + block_table_ptr + req * stride_bt_r + safe_lblk, mask=cached, other=0 + ).to(tl.int64) + cbase = ( + cache_ptr + + blk[:, None] * stride_c_blk + + head[None, :] * stride_c_h + + (safe_src % N)[:, None] * stride_c_n + + cd[None, :] * stride_c_d + ) + cv = tl.load(cbase, mask=c_mask[None, :] & cached[:, None], other=0.0).to( + tl.float32 + ) + wv = tl.load( + weight_ptr + coff * stride_w_d + iw * stride_w_w, mask=c_mask, other=0.0 + ).to(tl.float32) + acc += (xt + cv) * wv[None, :] + + if USE_SILU: + acc = acc * tl.sigmoid(acc) + if USE_RESIDUAL: + acc += xv.to(tl.float32) + + tl.store( + out_ptr + toff[:, None] * stride_x_t + coff[None, :], + acc.to(out_ptr.dtype.element_ty), + mask=tc_mask, + ) + + +def fused_sconv( + x: torch.Tensor, # [T, H*ws] head-major current-token inputs + weight: torch.Tensor, # [H*ws, W] + cache: torch.Tensor, # [num_blocks, H, N, D] paged + positions: torch.Tensor, # [T] int64 absolute position per token + block_table: torch.Tensor, # [num_reqs, max_blocks] int32 + seq_idx: torch.Tensor, # [T] int32 token -> batch request + slot_mapping: torch.Tensor, # [T] int64 flat slot (PAD = -1 => skip) + query_start: torch.Tensor, # [T] int32 first x-row of the token's request + off_s: int, + ws: int, + block_size: int, + activation: str | None = None, + use_residual: bool = True, +) -> torch.Tensor: + """Single-launch insert + depthwise causal conv1d over the paged cache. + + Reads same-forward taps from ``x`` and pre-forward taps from the cache, so + it is race-free in one launch for prefill / decode / spec and supports full + CUDA-graph capture for decode. + """ + T = x.shape[0] + out = torch.empty_like(x) + if T == 0: + return out + assert x.is_contiguous() + assert cache.stride(3) == 1, "cache D-dim must be contiguous" + H = cache.shape[1] + W = weight.shape[1] + C = H * ws # flat conv channel dim (all heads, head-major) + # Tile BT tokens x BLOCK_C channels per program: enough work per CTA to + # amortize the per-token addressing, while keeping the grid large for + # prefill. BLOCK_C spans heads so there is no per-head launch. A ~2K-element + # tile at 4 warps measured best on Blackwell; larger tiles spill registers. + BLOCK_C = min(triton.next_power_of_2(C), 256) + BT = 8 + grid = (triton.cdiv(T, BT), triton.cdiv(C, BLOCK_C)) + _fused_sconv_kernel[grid]( + x, + cache, + weight, + out, + positions, + seq_idx, + slot_mapping, + block_table, + query_start, + T, + x.stride(0), + cache.stride(0), + cache.stride(1), + cache.stride(2), + cache.stride(3), + weight.stride(0), + weight.stride(1), + block_table.stride(0), + block_table.shape[1], + block_size, + W=W, + USE_SILU=activation in ("silu", "swish"), + USE_RESIDUAL=use_residual, + OFF_S=off_s, + WS=ws, + H=H, + BT=BT, + BLOCK_C=BLOCK_C, + num_warps=4, + ) + return out + + +@triton.jit +def _seq_metadata_kernel( + qsl_ptr, # [num_reqs + 1] int32 cumulative query start rows + seq_idx_ptr, # [T] int32 out: token -> owning request + query_start_ptr, # [T] int32 out: first x-row of the token's request + num_reqs, + num_actual_tokens, + num_padded_tokens, + n_iters, # ceil(log2(num_reqs)): binary-search depth + BLOCK: tl.constexpr, +): + offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + tok = offs.to(tl.int32) + # Largest j in [0, num_reqs) with qsl[j] <= tok. + lo = tl.zeros([BLOCK], tl.int32) + hi = tl.full([BLOCK], num_reqs - 1, tl.int32) + for _ in range(n_iters): + mid = (lo + hi + 1) // 2 + below = tl.load(qsl_ptr + mid) <= tok + lo = tl.where(below, mid, lo) + hi = tl.where(below, hi, mid - 1) + actual = offs < num_actual_tokens + padded = offs < num_padded_tokens + query_start = tl.load(qsl_ptr + lo) + tl.store(seq_idx_ptr + offs, tl.where(actual, lo, 0), mask=padded) + tl.store( + query_start_ptr + offs, + tl.where(actual, query_start, 0), + mask=padded, + ) + + +def sconv_seq_metadata( + query_start_loc: torch.Tensor, + num_reqs: int, + num_actual_tokens: int, + seq_idx_out: torch.Tensor, + query_start_out: torch.Tensor, + num_padded_tokens: int | None = None, +) -> None: + """Fill static per-token seq_idx / query_start buffers in one launch. + + Replaces the arange + searchsorted + clamp + gather + 2x copy chain of the + sconv metadata build with a single kernel writing both persistent buffers. + Padded rows are filled with zero and must have ``slot_mapping == -1``. + """ + if num_padded_tokens is None: + num_padded_tokens = num_actual_tokens + if num_padded_tokens < num_actual_tokens: + raise ValueError("num_padded_tokens must cover all actual tokens") + if num_padded_tokens > seq_idx_out.shape[0]: + raise ValueError("seq_idx_out is too small for the padded token count") + if num_padded_tokens > query_start_out.shape[0]: + raise ValueError("query_start_out is too small for the padded token count") + + BLOCK = 256 + n_iters = (num_reqs - 1).bit_length() + grid = (triton.cdiv(num_padded_tokens, BLOCK),) + _seq_metadata_kernel[grid]( + query_start_loc, + seq_idx_out, + query_start_out, + num_reqs, + num_actual_tokens, + num_padded_tokens, + n_iters, + BLOCK=BLOCK, + ) diff --git a/vllm/models/inkling/nvidia/ops/silu_and_mul.py b/vllm/models/inkling/nvidia/ops/silu_and_mul.py new file mode 100644 index 0000000000..42ec25f632 --- /dev/null +++ b/vllm/models/inkling/nvidia/ops/silu_and_mul.py @@ -0,0 +1,197 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""SwiGLU kernels for the Inkling MLP layers. + +``silu_and_mul_triton``: SiLU-and-mul over the checkpoint's interleaved +fused gate/up layout (dense MLP). ``sink_silu_mul_epilogue``: the sink-expert +variant with the per-expert dequant scale and per-token gamma fused in. +""" + +import torch + +from vllm.triton_utils import tl, triton + + +@triton.jit(do_not_specialize=["M"]) +def _silu_and_mul_triton_kernel( + gateup_out_ptr, + down_inp_ptr, + M, + N: tl.constexpr, + GRID_SIZE: tl.constexpr, + NUM_STAGES: tl.constexpr, + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + EVEN_N: tl.constexpr, + INT64_INDEX: tl.constexpr, +): + start_pid = tl.program_id(0) + if INT64_INDEX: + start_pid = start_pid.to(tl.int64) + M = M.to(tl.int64) + + NUM_BLOCKS_N: tl.constexpr = tl.cdiv(N, BLOCK_SIZE_N) + num_blocks_mn = tl.cdiv(M, BLOCK_SIZE_M) * NUM_BLOCKS_N + + for pid in tl.range(start_pid, num_blocks_mn, GRID_SIZE, num_stages=NUM_STAGES): + pid_m = pid // NUM_BLOCKS_N + pid_n = pid % NUM_BLOCKS_N + + offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_n = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + mask_m = offs_m < M + mask_n = offs_n < N + + # Interleaved fused gate/up: [g0, u0, g1, u1, ...]. + mask_offs_2n = pid_n * BLOCK_SIZE_N + tl.arange(0, 2 * BLOCK_SIZE_N) // 2 + tl.static_assert(BLOCK_SIZE_N % 8 == 0, f"{BLOCK_SIZE_N=}") + mask_2n = mask_offs_2n < N + mask_2n = tl.max_constancy(mask_2n, [16]) + + offs_2n = pid_n * 2 * BLOCK_SIZE_N + tl.arange(0, 2 * BLOCK_SIZE_N) + offs_m2n = offs_m[:, None] * N * 2 + offs_2n[None, :] + + if EVEN_N or pid_n * BLOCK_SIZE_N + BLOCK_SIZE_N <= N: + gateup_out = tl.load( + gateup_out_ptr + offs_m2n, mask=mask_m[:, None], other=0.0 + ) + else: + mask_m2n = mask_m[:, None] & mask_2n[None, :] + gateup_out = tl.load(gateup_out_ptr + offs_m2n, mask=mask_m2n, other=0.0) + + gate_out, up_out = tl.split( + tl.reshape(gateup_out, (BLOCK_SIZE_M, BLOCK_SIZE_N, 2)) + ) + gate_out = gate_out.to(tl.float32) + up_out = up_out.to(tl.float32) + + down_inp = gate_out * tl.sigmoid(gate_out) * up_out + + mask_mn = mask_m[:, None] if EVEN_N else mask_m[:, None] & mask_n[None, :] + offs_mn = offs_m[:, None] * N + offs_n[None, :] + tl.store(down_inp_ptr + offs_mn, down_inp, mask=mask_mn) + + +def silu_and_mul_triton(gateup_output: torch.Tensor) -> torch.Tensor: + """SiLU-and-mul for the interleaved fused gate/up layout. + + Adapted from ``inkling_kernels.activation.silu_and_mul_fwd`` (without MXFP). + """ + assert gateup_output.is_contiguous(), ( + f"{gateup_output.shape=} {gateup_output.stride()=}" + ) + assert gateup_output.ndim == 2, f"{gateup_output.shape=}" + + M = gateup_output.shape[0] + hidden_size = gateup_output.shape[1] + assert hidden_size % 2 == 0, f"{hidden_size=}" + N = hidden_size // 2 + + down_input = torch.empty( + (M, N), device=gateup_output.device, dtype=gateup_output.dtype + ) + if M == 0: + return down_input + + BLOCK_SIZE_N = max(8, min(256, triton.next_power_of_2(N))) + if M <= 1: + BLOCK_SIZE_M = 4 + elif M <= 256: + BLOCK_SIZE_M = 2 + elif M < 4096: + BLOCK_SIZE_M = 4 + else: + BLOCK_SIZE_M = 16 + BLOCK_SIZE_N = max(8, min(128, triton.next_power_of_2(N))) + max_grid_size = triton.cdiv(M, BLOCK_SIZE_M) * triton.cdiv(N, BLOCK_SIZE_N) + num_sms = torch.cuda.get_device_properties( + gateup_output.device + ).multi_processor_count + grid_size = min(num_sms * 4, max_grid_size) + + _silu_and_mul_triton_kernel[(grid_size,)]( + gateup_out_ptr=gateup_output, + down_inp_ptr=down_input, + M=M, + N=N, + GRID_SIZE=grid_size, + NUM_STAGES=1, + BLOCK_SIZE_M=BLOCK_SIZE_M, + BLOCK_SIZE_N=BLOCK_SIZE_N, + EVEN_N=N % BLOCK_SIZE_N == 0, + INT64_INDEX=gateup_output.nbytes >= 2**31, + num_warps=8, + ) + + return down_input + + +@triton.jit(do_not_specialize=["T"]) +def _sink_epilogue_kernel( + raw_ptr, # [T, S * 2F] gemm1 output, interleaved g/u pairs per expert block + alpha_ptr, # [S] fp32 per-expert pre-SiLU dequant scale + gamma_ptr, # [T, S] fp32 per-token sink weights (may be strided) + ratio_ptr, # [S] fp32 per-expert post-SiLU scale (gemm2 alpha ratio) + out_ptr, # [T, S * F] output + T, + stride_raw_0, + stride_gamma_0, + F: tl.constexpr, + S: tl.constexpr, + BLOCK_F: tl.constexpr, +): + pid_t = tl.program_id(0).to(tl.int64) + pid_sf = tl.program_id(1) + if pid_t >= T: + return + s = pid_sf // (F // BLOCK_F) + offs_f = (pid_sf % (F // BLOCK_F)) * BLOCK_F + tl.arange(0, BLOCK_F) + + base = pid_t * stride_raw_0 + s * 2 * F + gate = tl.load(raw_ptr + base + 2 * offs_f).to(tl.float32) + up = tl.load(raw_ptr + base + 2 * offs_f + 1).to(tl.float32) + alpha = tl.load(alpha_ptr + s) + weight = tl.load(gamma_ptr + pid_t * stride_gamma_0 + s) * tl.load(ratio_ptr + s) + + gate *= alpha + up *= alpha + h = gate * tl.sigmoid(gate) * up * weight + tl.store(out_ptr + pid_t * (S * F) + s * F + offs_f, h) + + +def sink_silu_mul_epilogue( + raw: torch.Tensor, # [T, S * 2F] gemm1 output (interleaved gate/up rows) + alphas: torch.Tensor, # [S] fp32 + gammas: torch.Tensor, # [T, S] fp32 + ratios: torch.Tensor, # [S] fp32 + n_experts: int, + out_dtype: torch.dtype, +) -> torch.Tensor: + """Fused sink-expert epilogue: silu(g * a_e) * (u * a_e) * (gamma * r_e). + + One kernel replaces the per-expert dequant column scale, the SwiGLU, and + the per-token gamma multiply between the two sink GEMMs. + """ + tokens = raw.shape[0] + f = raw.shape[1] // (2 * n_experts) + out = torch.empty((tokens, n_experts * f), device=raw.device, dtype=out_dtype) + if tokens == 0: + return out + # raw may be a column-slice of a padded GEMM output (rows strided). + assert raw.stride(1) == 1 and gammas.stride(1) == 1 + # Largest power-of-two divisor of f (f = 768 -> 256), capped at 512. + block_f = min(512, f & (-f)) + _sink_epilogue_kernel[(tokens, n_experts * (f // block_f))]( + raw, + alphas, + gammas, + ratios, + out, + tokens, + raw.stride(0), + gammas.stride(0), + F=f, + S=n_experts, + BLOCK_F=block_f, + ) + return out diff --git a/vllm/models/inkling/nvidia/sconv_swa_attn.py b/vllm/models/inkling/nvidia/sconv_swa_attn.py new file mode 100644 index 0000000000..be27c5fd13 --- /dev/null +++ b/vllm/models/inkling/nvidia/sconv_swa_attn.py @@ -0,0 +1,221 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling short-conv state managed as a sliding-window KV cache. + +Each decoder layer owns one ``InklingConvState`` (an ``AttentionLayerBase``) that +emits a single ``SlidingWindowSpec`` for the layer's 4 sconv streams (K, V, +attn-output, mlp-output), packed head-major into one block: + + H = num_kv_heads (per-rank), N = block_size = sconv_kernel_size, + D = head_dim(K) + head_dim(V) + hidden/H(attn) + hidden/H(mlp) + +``D`` is TP-invariant; per rank we store ``H/TP`` heads of width ``D``. The conv +reads/writes this cache out-of-band via a custom backend; the (smaller) conv page +is padded up to the uniform attention page by ``unify_kv_cache_spec_page_size``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import ClassVar + +import torch +from torch import nn + +from vllm.config import VllmConfig, get_current_vllm_config +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.v1.attention.backend import ( + AttentionBackend, + AttentionCGSupport, + AttentionMetadata, + AttentionMetadataBuilder, + CommonAttentionMetadata, +) +from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, SlidingWindowSpec + +from .ops.sconv import sconv_seq_metadata + +# Stream order within the per-head packed D (== contiguous sub-ranges). +_K, _V, _ATTN, _MLP = 0, 1, 2, 3 + + +@dataclass +class InklingSconvMetadata(AttentionMetadata): + block_table: torch.Tensor # [num_reqs, max_blocks] physical blocks per req + slot_mapping: torch.Tensor # [T] int64 flat slot of each token (-1 => skip) + seq_idx: torch.Tensor # [T] int32 token -> batch request + query_start: torch.Tensor # [T] int32 first x-row of each token's request + + +class InklingSconvMetadataBuilder(AttentionMetadataBuilder[InklingSconvMetadata]): + _cudagraph_support: ClassVar[AttentionCGSupport] = ( + AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE + ) + + def __init__( + self, + kv_cache_spec: AttentionSpec, + layer_names: list[str], + vllm_config: VllmConfig, + device: torch.device, + ) -> None: + super().__init__(kv_cache_spec, layer_names, vllm_config, device) + assert isinstance(kv_cache_spec, SlidingWindowSpec) + # Persistent per-token buffers for CUDA graph capture. + max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens + self.seq_idx_buffer = torch.empty( + max_num_tokens, dtype=torch.int32, device=device + ) + self.query_start_buffer = torch.empty( + max_num_tokens, dtype=torch.int32, device=device + ) + + def build( + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + fast_build: bool = False, + ) -> InklingSconvMetadata: + num_reqs = common_attn_metadata.num_reqs + num_actual_tokens = int(common_attn_metadata.query_start_loc_cpu[-1]) + num_padded_tokens = common_attn_metadata.slot_mapping.shape[0] + assert num_padded_tokens >= num_actual_tokens + + # Per-token seq_idx (owning request) and query_start (first x-row of + # that request; the fused kernel uses it to tell same-forward taps, + # read from x, from pre-forward taps, read from cache) in one launch. + sconv_seq_metadata( + common_attn_metadata.query_start_loc, + num_reqs, + num_actual_tokens, + self.seq_idx_buffer, + self.query_start_buffer, + num_padded_tokens, + ) + + return InklingSconvMetadata( + block_table=common_attn_metadata.block_table_tensor, + slot_mapping=common_attn_metadata.slot_mapping[:num_padded_tokens], + seq_idx=self.seq_idx_buffer[:num_padded_tokens], + query_start=self.query_start_buffer[:num_padded_tokens], + ) + + +class InklingSconvBackend(AttentionBackend): + """Custom dummy backend for the sconv sliding-window cache management.""" + + @staticmethod + def get_name() -> str: + return "INKLING_SCONV_SWA" + + @classmethod + def indexes_kv_by_block_stride(cls) -> bool: + # num_blocks is the outermost dim (HND, see get_kv_cache_shape), so the + # padded conv page is read through a strided view. + return True + + @staticmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + cache_dtype_str: str = "auto", + ) -> tuple[int, ...]: + # HND, num-blocks-first, head-major: [num_blocks, H, N, D]. + return (num_blocks, num_kv_heads, block_size, head_size) + + @staticmethod + def get_kv_cache_stride_order( + include_num_layers_dimension: bool = False, + ) -> tuple[int, ...]: + # Identity: physical layout == logical [num_blocks, H, N, D]. + if include_num_layers_dimension: + return (0, 1, 2, 3, 4) + return (0, 1, 2, 3) + + @staticmethod + def get_impl_cls(): + raise NotImplementedError( + "InklingSconvBackend has no attention impl; the conv runs out-of-band." + ) + + @staticmethod + def get_builder_cls() -> type[InklingSconvMetadataBuilder]: + return InklingSconvMetadataBuilder + + +class InklingConvState(nn.Module, AttentionLayerBase): + """Per-decoder-layer owner emitting one sliding-window conv-state spec.""" + + def __init__( + self, + *, + num_kv_heads: int, + head_dim: int, + hidden_size: int, + kernel_size: int, + prefix: str, + ) -> None: + super().__init__() + self.prefix = prefix + # Bound to the manager-allocated paged cache by bind_kv_cache; a + # placeholder until then. Read out-of-band by InklingShortConv. + self.kv_cache = torch.tensor([]) + tp_size = get_tensor_model_parallel_world_size() + # Guardrails for the conv-state layout below; only these are exercised. + # tp_size <= num_kv_heads keeps >=1 whole KV head per rank (no + # replication/clamping), so the per-head width stays TP-invariant. + assert tp_size <= num_kv_heads, ( + f"sconv SWA cache supports tp_size <= num_kv_heads ({num_kv_heads}), " + f"got {tp_size}" + ) + # Per-rank head count; D is TP-invariant (K/V heads and the hidden + # chunk both scale 1/TP together). The attn-/mlp-output sconv streams + # are hidden-sharded: each rank owns its H/tp chunk (the sublayer + # outputs are reduce-scattered / all-gathered around the conv). + self.num_kv_heads = num_kv_heads // tp_size + hidden_per_head = hidden_size // num_kv_heads + # Packed per-head width: K + V + attn-output chunk + mlp-output chunk, + # padded to a power of two so every layer's conv page is the same size + # and an exact multiple of the attention page (the page unifier then + # scales attention block sizes instead of padding). + raw_head_size = 2 * head_dim + 2 * hidden_per_head + self.head_size = 1 << (raw_head_size - 1).bit_length() + self.sliding_window = kernel_size + self.block_size = kernel_size + # Per-head D-sub-range (offset, width) for each stream. Streams share + # the cache; each writes/reads its own width across all H heads. + self.stream_ranges: tuple[tuple[int, int], ...] = ( + (0, head_dim), # _K + (head_dim, head_dim), # _V + (2 * head_dim, hidden_per_head), # _ATTN + (2 * head_dim + hidden_per_head, hidden_per_head), # _MLP + ) + vllm_config = get_current_vllm_config() + self._dtype = vllm_config.model_config.dtype + assert self._dtype == torch.bfloat16, ( + f"sconv SWA cache supports bfloat16 only, got {self._dtype}" + ) + # Register in the forward context so the runner enumerates this owner as + # an attention-like layer (get_kv_cache_spec / get_attn_backend). + compilation_config = vllm_config.compilation_config + if prefix in compilation_config.static_forward_context: + raise ValueError(f"Duplicate layer name: {prefix}") + compilation_config.static_forward_context[prefix] = self + + def forward(self): ... + + def get_attn_backend(self) -> type[AttentionBackend]: + return InklingSconvBackend + + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: + return SlidingWindowSpec( + block_size=self.block_size, + num_kv_heads=self.num_kv_heads, + head_size=self.head_size, + head_size_v=0, # all 4 streams packed into head_size + dtype=self._dtype, + sliding_window=self.sliding_window, + ) diff --git a/vllm/models/inkling/nvidia/short_conv.py b/vllm/models/inkling/nvidia/short_conv.py new file mode 100644 index 0000000000..69d5a26374 --- /dev/null +++ b/vllm/models/inkling/nvidia/short_conv.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling short convolution: depthwise causal conv1d (+ residual) over a paged +sliding-window conv-state cache. + +Each decoder layer owns one ``InklingConvState`` (``sconv_swa_attn.py``) holding the +manager-allocated paged cache for the layer's 4 sconv streams (K, V, attn-output, +mlp-output), packed head-major into one block. Each ``InklingShortConv`` is a +stateless weight + kernel launcher that, per forward (positions-addressed, the +same path for prefill / decode / mixed), inserts the current tokens' inputs +into their paged slot and convolves each token against the ``W`` taps ending +at its absolute position, reading pre-forward window positions out of the +paged cache via the block table. + +Per-forward metadata (``block_table`` / ``slot_mapping`` / ``seq_idx`` / +``query_start``) is built once by ``InklingSconvMetadataBuilder`` and published under +the owner's prefix in the forward context; the absolute ``positions`` are +threaded in from the model. The insert + conv run in a single ``fused_sconv`` +launch (same path for prefill / decode / mixed / spec). All inputs are +fixed-address persistent buffers and the grid is fixed, so decode can replay +under a full CUDA graph. +""" + +from __future__ import annotations + +import torch +from torch import nn +from torch.nn.parameter import Parameter + +from vllm.distributed import get_tensor_model_parallel_rank +from vllm.forward_context import get_forward_context +from vllm.model_executor.utils import set_weight_attrs + +from .ops import fused_sconv +from .sconv_swa_attn import InklingConvState, InklingSconvMetadata + + +class InklingShortConv(nn.Module): + def __init__( + self, dim: int, kernel_size: int, owner: InklingConvState, stream_idx: int + ) -> None: + super().__init__() + self.dim = dim + self.kernel_size = kernel_size + self.owner = owner + self.stream_idx = stream_idx + self.tp_rank = get_tensor_model_parallel_rank() + + # Depthwise conv weight; checkpoint stores (dim, 1, W). + self.weight = Parameter(torch.empty(dim, 1, kernel_size), requires_grad=False) + set_weight_attrs(self.weight, {"weight_loader": self.weight_loader}) + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor) -> None: + if loaded_weight.shape[0] != param.shape[0]: + shard = param.shape[0] + loaded_weight = loaded_weight.narrow(0, self.tp_rank * shard, shard) + param.data.copy_(loaded_weight) + + def forward(self, x: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + # x: (num_tokens, dim); positions: (num_tokens,) absolute positions. + attn_metadata = get_forward_context().attn_metadata + if not isinstance(attn_metadata, dict): + # Memory-profiling / no metadata: identity (residual). + return x + m = attn_metadata.get(self.owner.prefix) + if m is None: + return x + assert isinstance(m, InklingSconvMetadata) + cache = self.owner.kv_cache + if cache.numel() == 0: + # Cache not yet bound (profiling before KV alloc): identity. + return x + + off_s, ws = self.owner.stream_ranges[self.stream_idx] + block_size = self.owner.block_size + x = x.contiguous() + weight = self.weight.squeeze(1) # (dim, W) + + return fused_sconv( + x, + weight, + cache, + positions, + m.block_table, + m.seq_idx, + m.slot_mapping, + m.query_start, + off_s, + ws, + block_size, + activation=None, + use_residual=True, + ) diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index fc2d653779..895f7516a5 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -110,6 +110,8 @@ class ParserEngine(Parser): self._has_reasoning = ( "THINK_END" in parser_engine_config.token_id_terminals + or "THINK_START" in parser_engine_config.terminals + or "THINK_END" in parser_engine_config.terminals or parser_engine_config.initial_state == ParserState.REASONING ) self._reasoning_ended: bool = not self._has_reasoning diff --git a/vllm/parser/engine/parser_engine_config.py b/vllm/parser/engine/parser_engine_config.py index f18f3f02e1..ad83e33149 100644 --- a/vllm/parser/engine/parser_engine_config.py +++ b/vllm/parser/engine/parser_engine_config.py @@ -28,6 +28,7 @@ from vllm.parser.engine.events import EventType class ParserState(Enum): CONTENT = auto() REASONING = auto() + MESSAGE_HEADER = auto() TOOL_PREAMBLE = auto() TOOL_NAME = auto() TOOL_ARGS = auto() diff --git a/vllm/parser/engine/registered_adapters.py b/vllm/parser/engine/registered_adapters.py index 889c71504f..d259ade545 100644 --- a/vllm/parser/engine/registered_adapters.py +++ b/vllm/parser/engine/registered_adapters.py @@ -12,6 +12,7 @@ from vllm.parser.deepseek_v32 import DeepSeekV32Parser from vllm.parser.engine.adapters import make_adapters from vllm.parser.gemma4 import Gemma4Parser from vllm.parser.glm47_moe import Glm47MoeParser +from vllm.parser.inkling import InklingParser from vllm.parser.kimi_k2 import KimiK2Parser from vllm.parser.minimax_m2 import MinimaxM2Parser from vllm.parser.nemotron_v3 import NemotronV3Parser @@ -62,3 +63,8 @@ from vllm.parser.seed_oss import SeedOssParser KimiK2ParserReasoningAdapter, KimiK2ParserToolAdapter, ) = make_adapters(KimiK2Parser) + +( + InklingParserReasoningAdapter, + InklingParserToolAdapter, +) = make_adapters(InklingParser) diff --git a/vllm/parser/engine/streaming_parser_engine.py b/vllm/parser/engine/streaming_parser_engine.py index 319791961c..8cafdf8e62 100644 --- a/vllm/parser/engine/streaming_parser_engine.py +++ b/vllm/parser/engine/streaming_parser_engine.py @@ -269,6 +269,8 @@ class StreamingParserEngine: SemanticEvent(EventType.REASONING_END, tool_index=self.tool_index) ) self.state = ParserState.CONTENT + elif self.state == ParserState.MESSAGE_HEADER: + self.state = ParserState.CONTENT return events @@ -314,6 +316,15 @@ class StreamingParserEngine: return self._emit_for_state(value) if self.skip_tool_parsing and terminal in self._tool_terminals: + if self.state == ParserState.MESSAGE_HEADER: + self.state = ParserState.CONTENT + return [ + SemanticEvent( + EventType.TEXT_CHUNK, + value=value, + tool_index=self.tool_index, + ) + ] if EventType.REASONING_END in transition.events: self.state = ParserState.CONTENT return [ diff --git a/vllm/parser/inkling.py b/vllm/parser/inkling.py new file mode 100644 index 0000000000..84502be018 --- /dev/null +++ b/vllm/parser/inkling.py @@ -0,0 +1,411 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling parser: typed content blocks parsed by a single state machine. + +Inkling output format (every marker is a dedicated special token):: + + <|message_model|><|content_thinking|>...reasoning...<|end_message|> + <|message_model|><|content_text|>...visible text...<|end_message|> + <|message_model|><|content_invoke_tool_json|> + {"name":"get_weather","args":{"city":"SF"}}<|end_message|> + +Blocks are self-describing and may repeat in any order; sampling may +also end a block with the standalone ``<|content_model_end_sampling|>`` +token. The tool-call payload is a single JSON object whose ``name`` is +extracted by the engine's name-from-args path and whose ``args`` object +is carved out of the wrapper by :func:`_inkling_arg_converter`. + +Note the terminal *labels*: ``THINK_START``/``THINK_END`` are what the +engine keys its reasoning plumbing on (``is_reasoning_end``, +``count_reasoning_tokens``, initial-state seeding), so ``<|end_message|>`` +is labelled ``THINK_END`` here even though it ends every block kind — +the transition table, not the label, carries the semantics. +""" + +from __future__ import annotations + +import functools +import json +from collections.abc import Sequence +from typing import TYPE_CHECKING + +from vllm.entrypoints.openai.engine.protocol import ( + ExtractedToolCallInformation, +) +from vllm.parser.engine.events import EventType +from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, +) + +if TYPE_CHECKING: + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + +MESSAGE_MODEL = "<|message_model|>" +CONTENT_TEXT = "<|content_text|>" +CONTENT_THINKING = "<|content_thinking|>" +CONTENT_INVOKE_TOOL_JSON = "<|content_invoke_tool_json|>" +CONTENT_INVOKE_TOOL_TEXT = "<|content_invoke_tool_text|>" +CONTENT_TOOL_ERROR = "<|content_tool_error|>" +CONTENT_MODEL_END_SAMPLING = "<|content_model_end_sampling|>" +END_MESSAGE = "<|end_message|>" + +INKLING_SPECIAL_TOKENS = ( + MESSAGE_MODEL, + CONTENT_TEXT, + CONTENT_THINKING, + CONTENT_INVOKE_TOOL_JSON, + CONTENT_INVOKE_TOOL_TEXT, + CONTENT_TOOL_ERROR, + CONTENT_MODEL_END_SAMPLING, + END_MESSAGE, +) + +_WS = " \t\r\n" + + +def _scan_json_value(raw: str, start: int) -> int | None: + """Return the end index (exclusive) of the JSON object starting at + ``raw[start]``, or ``None`` when the object is still unterminated.""" + depth = 0 + in_string = False + escape = False + for i in range(start, len(raw)): + ch = raw[i] + if escape: + escape = False + continue + if in_string: + if ch == "\\": + escape = True + elif ch == '"': + in_string = False + continue + if ch == '"': + in_string = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return i + 1 + return None + + +def _args_value_span(raw: str) -> str | None: + """Extract the raw text span of the top-level ``"args"`` value from a + (possibly incomplete) ``{"name":...,"args":{...}}`` wrapper. + + Returns the verbatim substring (prefix-stable across growing input, + which the engine's argument-delta diffing relies on), possibly an + unterminated object prefix; ``None`` when the value has not started. + Raises ``ValueError`` when the value is not a JSON object. + """ + depth = 0 + in_string = False + escape = False + string_start = -1 + last_string: str | None = None + for i, ch in enumerate(raw): + if escape: + escape = False + continue + if in_string: + if ch == "\\": + escape = True + elif ch == '"': + in_string = False + if depth == 1: + last_string = raw[string_start + 1 : i] + continue + if ch == '"': + in_string = True + string_start = i + elif ch == ":" and depth == 1 and last_string == "args": + value_start = i + 1 + while value_start < len(raw) and raw[value_start] in _WS: + value_start += 1 + if value_start >= len(raw): + return None + if raw[value_start] != "{": + raise ValueError("Inkling tool call args must be a JSON object") + value_end = _scan_json_value(raw, value_start) + if value_end is None: + return raw[value_start:] + return raw[value_start:value_end] + elif ch in "{[": + depth += 1 + elif ch in "}]": + depth -= 1 + return None + + +def _inkling_arg_converter(raw_args: str, partial: bool) -> str: + """Carve the ``args`` object out of the tool-call JSON wrapper. + + Why a converter at all: the engine's ``tool_args_json`` machinery + treats the *entire* TOOL_ARGS text as the tool arguments, but Inkling's + payload is the ``{"name":...,"args":{...}}`` wrapper — without a + converter, ``_compute_arg_delta`` streams the wrapper verbatim into + the OpenAI ``arguments`` field (``converter is None -> raw delta``, + unconditionally; ``stream_arg_deltas=False`` does not stop it). + + Why a hand-rolled scanner instead of (partial) ``json.loads`` + + ``json.dumps``: the engine diffs successive converter outputs and + requires each to extend the previous one (``startswith``); a + violation silently drops argument deltas. Re-serialization changes + whitespace and closes unterminated structures differently across + ticks, so the only prefix-stable output is a verbatim substring of + the input. The scanner is also string/escape-aware so an ``"args"`` + literal inside the name or a string value cannot mislead it, and it + still recovers a partial span when EOS truncates the wrapper (where + ``json.loads`` would fail). + """ + span = _args_value_span(raw_args) + if span is None: + # No args value yet (streaming) or none at all (treat as empty). + return "" if partial else "{}" + return span + + +@functools.cache +def inkling_config() -> ParserEngineConfig: + terminals = { + "MSG_MODEL": MESSAGE_MODEL, + "TEXT_START": CONTENT_TEXT, + "THINK_START": CONTENT_THINKING, + "THINK_END": END_MESSAGE, + "END_SAMPLING": CONTENT_MODEL_END_SAMPLING, + "TOOL_START": CONTENT_INVOKE_TOOL_JSON, + "TOOL_TEXT": CONTENT_INVOKE_TOOL_TEXT, + "TOOL_ERROR": CONTENT_TOOL_ERROR, + } + transitions: dict[tuple[ParserState, str], Transition] = { + # ── Between blocks / inside a text block ────────────────────── + (ParserState.CONTENT, "MSG_MODEL"): Transition( + ParserState.MESSAGE_HEADER, + (), + ), + (ParserState.CONTENT, "TEXT_START"): Transition( + ParserState.CONTENT, + (), + ), + (ParserState.CONTENT, "THINK_START"): Transition( + ParserState.REASONING, + (EventType.REASONING_START,), + ), + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_ARGS, + (EventType.TOOL_CALL_START,), + ), + # Raw / error tool blocks render as visible text. + (ParserState.CONTENT, "TOOL_TEXT"): Transition( + ParserState.CONTENT, + (), + ), + (ParserState.CONTENT, "TOOL_ERROR"): Transition( + ParserState.CONTENT, + (), + ), + # The optional function name between the model-role and content-kind + # markers is metadata, not visible assistant content. + (ParserState.MESSAGE_HEADER, "MSG_MODEL"): Transition( + ParserState.MESSAGE_HEADER, + (), + ), + (ParserState.MESSAGE_HEADER, "TEXT_START"): Transition( + ParserState.CONTENT, + (), + ), + (ParserState.MESSAGE_HEADER, "THINK_START"): Transition( + ParserState.REASONING, + (EventType.REASONING_START,), + ), + (ParserState.MESSAGE_HEADER, "TOOL_START"): Transition( + ParserState.TOOL_ARGS, + (EventType.TOOL_CALL_START,), + ), + (ParserState.MESSAGE_HEADER, "TOOL_TEXT"): Transition( + ParserState.CONTENT, + (), + ), + (ParserState.MESSAGE_HEADER, "TOOL_ERROR"): Transition( + ParserState.CONTENT, + (), + ), + # ── Inside a thinking block ─────────────────────────────────── + (ParserState.REASONING, "THINK_START"): Transition( + ParserState.REASONING, + (), + ), + # Defensive: tool call opening while a thinking block is still + # unclosed. + (ParserState.REASONING, "TOOL_START"): Transition( + ParserState.TOOL_ARGS, + (EventType.REASONING_END, EventType.TOOL_CALL_START), + ), + } + # Block-end terminals behave identically regardless of label. A + # closed tool block returns to CONTENT (Inkling has no section wrapper; + # blocks of any kind may follow), which also keeps the block-kind + # and role tokens out of the engine's tool-terminal set so the + # skip_tool_parsing reasoning pass still classifies reasoning. + for end in ("THINK_END", "END_SAMPLING"): + transitions[(ParserState.CONTENT, end)] = Transition( + ParserState.CONTENT, + (), + ) + transitions[(ParserState.REASONING, end)] = Transition( + ParserState.CONTENT, + (EventType.REASONING_END,), + ) + transitions[(ParserState.TOOL_ARGS, end)] = Transition( + ParserState.CONTENT, + (EventType.TOOL_CALL_END,), + ) + transitions[(ParserState.MESSAGE_HEADER, end)] = Transition( + ParserState.CONTENT, + (), + ) + + return ParserEngineConfig( + name="inkling", + # Normal generation continues after a prompt-prefilled + # `<|message_model|>`. Non-streaming parsing receives only the generated + # suffix, so begin in the corresponding message-header state as well. + initial_state=ParserState.MESSAGE_HEADER, + terminals=terminals, + # Inkling content-kind markers are the grammar. When the engine is + # used through DelegatingParser, the reasoning pass can hand the tool + # pass reconstructed text whose token-id slice no longer contains the + # content-kind marker that starts the tool block, so keep Inkling on + # the text grammar instead of token-id-only terminal matching. + token_id_terminals={}, + transitions=transitions, + arg_converter=_inkling_arg_converter, + stream_arg_deltas=True, + tool_args_json=True, + strip_trailing_reasoning_whitespace=True, + drop_whitespace_only_content_before_tools=True, + strip_content_whitespace_with_tools=False, + validate_tool_names=False, + ) + + +class InklingParser(ParserEngine): + CONFIG_NAME = "inkling" + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + **kwargs, + ) -> None: + kwargs.setdefault("parser_engine_config", inkling_config()) + super().__init__(tokenizer, tools, **kwargs) + + def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None: + """Seed the initial parsing state from the prompt tail. + + Mirrors the Rust parser's ``initialize()``: scanning the prompt + backwards, the last relevant special token decides whether generation + continues inside a thinking block, a text block, a model message + header, or between blocks. + """ + vocab = self.vocab + thinking_id = vocab.get(CONTENT_THINKING) + text_id = vocab.get(CONTENT_TEXT) + model_id = vocab.get(MESSAGE_MODEL) + special_ids = {vocab[text] for text in INKLING_SPECIAL_TOKENS if text in vocab} + for token_id in reversed(prompt_token_ids): + if token_id == thinking_id: + self._engine.reset(initial_state=ParserState.REASONING) + self._streaming_initialized = True + return + if token_id == text_id: + self._engine.reset(initial_state=ParserState.CONTENT) + self._streaming_initialized = True + return + if token_id == model_id: + self._engine.reset(initial_state=ParserState.MESSAGE_HEADER) + self._streaming_initialized = True + return + if token_id in special_ids: + break + + def is_reasoning_end(self, input_ids: list[int]) -> bool: + vocab = self.vocab + thinking_id = vocab.get(CONTENT_THINKING) + text_id = vocab.get(CONTENT_TEXT) + model_id = vocab.get(MESSAGE_MODEL) + end_sampling_id = vocab.get(CONTENT_MODEL_END_SAMPLING) + for token_id in reversed(input_ids): + if token_id in (thinking_id, model_id): + return False + if token_id in (text_id, end_sampling_id): + return True + return False + + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + vocab = self.vocab + thinking_id = vocab.get(CONTENT_THINKING) + end_ids = { + token_id + for token_id in ( + vocab.get(END_MESSAGE), + vocab.get(CONTENT_MODEL_END_SAMPLING), + ) + if token_id is not None + } + in_reasoning = False + count = 0 + for token_id in token_ids: + if token_id == thinking_id: + in_reasoning = True + continue + if token_id in end_ids: + in_reasoning = False + continue + if in_reasoning: + count += 1 + return count + + def _single_pass_parse( + self, + text: str, + token_ids: Sequence[int], + initial_state: ParserState | None = None, + ) -> tuple[str | None, str | None, ExtractedToolCallInformation]: + reasoning, content, tool_call_info = super()._single_pass_parse( + text, token_ids, initial_state=initial_state + ) + # The engine defers content that follows tool-call events within a + # single pass; Inkling allows text blocks after tool-call blocks, so + # flush the trailing text (matching the Rust unified parser). + if self._deferred_content: + trailing = self._deferred_content + self._deferred_content = "" + content = self._strip_content_whitespace( + (content or "") + trailing, + tool_call_info.tools_called, + ) + tool_call_info = ExtractedToolCallInformation( + tools_called=tool_call_info.tools_called, + tool_calls=tool_call_info.tool_calls, + content=content, + ) + return reasoning, content, tool_call_info + + @staticmethod + def _extract_args_value(parsed: dict) -> str | None: + # Inkling wraps arguments under "args" rather than "arguments". + for key in ("args", "arguments", "parameters"): + if key in parsed: + val = parsed[key] + if isinstance(val, str): + return val + return json.dumps(val, ensure_ascii=False) + return None diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index ba5ed47185..84682cdc8b 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -128,6 +128,10 @@ _REASONING_PARSERS_TO_REGISTER = { "step3p5_reasoning_parser", "Step3p5ReasoningParser", ), + "inkling": ( + "inkling_reasoning_parser", + "InklingParserReasoningAdapter", + ), } diff --git a/vllm/reasoning/inkling_reasoning_parser.py b/vllm/reasoning/inkling_reasoning_parser.py new file mode 100644 index 0000000000..b9246e6c58 --- /dev/null +++ b/vllm/reasoning/inkling_reasoning_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import InklingParserReasoningAdapter + +__all__ = ["InklingParserReasoningAdapter"] diff --git a/vllm/renderers/inkling.py b/vllm/renderers/inkling.py new file mode 100644 index 0000000000..87eedfd69d --- /dev/null +++ b/vllm/renderers/inkling.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Native Inkling chat renderer for the Python frontend. + +Mirrors the Rust frontend's native Inkling renderer +(``rust/src/chat/src/renderer/inkling/mod.rs``): chat messages are rendered +directly to token ids — Inkling has no Jinja chat template and no faithful +text form. + +The encoding logic lives in ``inkling_encoding.py`` behind a narrow +"OpenAI messages + tools -> token ids" call; see the swap-point comment +in :meth:`InklingRenderer._render` for adopting a standalone Inkling +input-processing library (mistral-common style) later. +""" + +from vllm.config import VllmConfig +from vllm.entrypoints.chat_utils import ( + ChatCompletionMessageParam, + ConversationMessage, + parse_chat_messages, + parse_chat_messages_async, +) +from vllm.logger import init_logger +from vllm.tokenizers.hf import HfTokenizer +from vllm.utils.async_utils import make_async + +from .base import BaseRenderer +from .inkling_encoding import SPECIAL_TOKEN_SPELLINGS, render_inkling_messages +from .inputs import DictPrompt +from .inputs.preprocess import parse_dec_only_prompt +from .params import ChatParams + +logger = init_logger(__name__) + +_NAMED_REASONING_EFFORT = { + "none": 0.0, + "minimal": 0.1, + "low": 0.2, + "medium": 0.7, + "high": 0.9, + "xhigh": 0.99, + "max": 0.99, +} + +_DEFAULT_REASONING_EFFORT = 0.9 + + +def _resolve_reasoning_effort(value: object) -> float | int | None: + if value is None: + return _DEFAULT_REASONING_EFFORT + if isinstance(value, str): + return _NAMED_REASONING_EFFORT.get(value) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return value + + +class _HfBackedTmlTokenizer: + """Adapts an HF tokenizer to the encoding core's tokenizer protocol. + + Special-token ids are resolved from the tokenizer vocab at + construction (never hardcoded), trying each known spelling — the Inkling + HF vocab exposes some semantic slots as ``<|unused_NNNNNN|>`` tokens. + """ + + def __init__(self, tokenizer: HfTokenizer) -> None: + self._tokenizer = tokenizer + + vocab = tokenizer.get_vocab() + special_ids: dict[str, int] = {} + missing: list[str] = [] + for token, spellings in SPECIAL_TOKEN_SPELLINGS.items(): + for spelling in spellings: + token_id = vocab.get(spelling) + if token_id is not None: + special_ids[token] = token_id + break + else: + missing.append(token) + if missing: + raise ValueError(f"Inkling tokenizer is missing special tokens: {missing}") + self._special_ids = special_ids + + def encode_text(self, text: str) -> list[int]: + return self._tokenizer.encode(text, add_special_tokens=False) + + def encode_special(self, token: str) -> int: + return self._special_ids[token] + + +class InklingRenderer(BaseRenderer[HfTokenizer]): + def __init__( + self, + config: VllmConfig, + tokenizer: HfTokenizer | None, + ) -> None: + super().__init__(config, tokenizer) + + self._inkling_tokenizer = _HfBackedTmlTokenizer(self.get_tokenizer()) + self._render_async = make_async(self._render, executor=self._executor) + + def _render( + self, + messages: list[ChatCompletionMessageParam], + params: ChatParams, + ) -> list[int]: + kwargs = params.chat_template_kwargs or {} + + if kwargs.get("continue_final_message"): + raise ValueError("Inkling renderer does not support continue_final_message") + + reasoning_effort = _resolve_reasoning_effort(kwargs.get("reasoning_effort")) + + try: + # Swap point: to adopt a standalone Inkling input-processing + # library, replace this call (and the _HfBackedTmlTokenizer + # adapter above) with the library's renderer. + return render_inkling_messages( + messages, + self._inkling_tokenizer, + add_generation_prompt=kwargs.get("add_generation_prompt", True), + tools=kwargs.get("tools"), + reasoning_effort=reasoning_effort, + ) + except ValueError: + raise + except (TypeError, KeyError) as e: + # Malformed request content; surface as a request error. + raise ValueError(str(e)) from e + except Exception as e: + logger.exception("Error while rendering Inkling chat messages") + raise ValueError(str(e)) from e + + def render_messages( + self, + messages: list[ChatCompletionMessageParam], + params: ChatParams, + ) -> tuple[list[ConversationMessage], DictPrompt]: + conversation, mm_data, mm_uuids = parse_chat_messages( + messages, + self.model_config, + content_format="string", + media_io_kwargs=params.media_io_kwargs, + mm_processor_kwargs=params.mm_processor_kwargs, + ) + + token_ids = self._render(messages, params) + + prompt = parse_dec_only_prompt(token_ids) + if mm_data is not None: + prompt["multi_modal_data"] = mm_data + if mm_uuids is not None: + prompt["multi_modal_uuids"] = mm_uuids + + return conversation, prompt + + async def render_messages_async( + self, + messages: list[ChatCompletionMessageParam], + params: ChatParams, + ) -> tuple[list[ConversationMessage], DictPrompt]: + conversation, mm_data, mm_uuids = await parse_chat_messages_async( + messages, + self.model_config, + content_format="string", + media_io_kwargs=params.media_io_kwargs, + mm_processor_kwargs=params.mm_processor_kwargs, + ) + + token_ids = await self._render_async(messages, params) + + prompt = parse_dec_only_prompt(token_ids) + if mm_data is not None: + prompt["multi_modal_data"] = mm_data + if mm_uuids is not None: + prompt["multi_modal_uuids"] = mm_uuids + + return conversation, prompt diff --git a/vllm/renderers/inkling_encoding.py b/vllm/renderers/inkling_encoding.py new file mode 100644 index 0000000000..38039e4232 --- /dev/null +++ b/vllm/renderers/inkling_encoding.py @@ -0,0 +1,383 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inkling chat-encoding core. + +Pure implementation of Inkling chat rendering, kept deliberately free of +vLLM imports: it depends only on the :class:`InklingTextTokenizer` protocol +and speaks OpenAI-style message dicts. If a standalone Inkling +input-processing library becomes available, this module is the unit to +swap out — the swap point is marked in ``vllm/renderers/inkling.py``. + +Multimodal parts follow the contract of vLLM's Inkling multimodal processor +(``InklingMultiModalProcessor`` anchors prompt updates on the bare +content-kind marker and inserts the per-patch placeholder run itself): + +* image parts emit only ``<|content_image|>`` — no seed placeholder id; +* audio parts emit ``<|content_audio_input|><|audio_end|>`` — no seed + placeholder id between them. + +``reasoning_effort`` (a float in [0, 0.99], sourced from +``chat_template_kwargs`` only) renders the ``Thinking effort level:`` +system control block after the tool declarations and initial system messages. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterator, Mapping, Sequence +from typing import Any, Protocol + +END_OF_TEXT = "<|endoftext|>" +MESSAGE_USER = "<|message_user|>" +MESSAGE_MODEL = "<|message_model|>" +MESSAGE_SYSTEM = "<|message_system|>" +MESSAGE_TOOL = "<|message_tool|>" +CONTENT_TEXT = "<|content_text|>" +CONTENT_IMAGE = "<|content_image|>" +CONTENT_MODEL_END_SAMPLING = "<|content_model_end_sampling|>" +CONTENT_THINKING = "<|content_thinking|>" +CONTENT_AUDIO_INPUT = "<|content_audio_input|>" +CONTENT_TOOL_ERROR = "<|content_tool_error|>" +CONTENT_XML = "<|content_xml|>" +CONTENT_INVOKE_TOOL_JSON = "<|content_invoke_tool_json|>" +CONTENT_INVOKE_TOOL_TEXT = "<|content_invoke_tool_text|>" +END_MESSAGE = "<|end_message|>" +AUDIO_END = "<|audio_end|>" + +ROLE_MESSAGE_TOKENS: dict[str, str] = { + "user": MESSAGE_USER, + "assistant": MESSAGE_MODEL, + "system": MESSAGE_SYSTEM, + # The developer role folds into the system role on the Inkling wire. + "developer": MESSAGE_SYSTEM, + "tool": MESSAGE_TOOL, +} + +# Alternate vocab spellings per semantic token. The Inkling HF tokenizer +# exposes some semantic slots as ``<|unused_NNNNNN|>`` tokens (notably +# CONTENT_XML = <|unused_200024|>), so ID resolution must try these +# spellings in order. +SPECIAL_TOKEN_SPELLINGS: dict[str, tuple[str, ...]] = { + MESSAGE_USER: (MESSAGE_USER,), + MESSAGE_MODEL: (MESSAGE_MODEL,), + MESSAGE_SYSTEM: (MESSAGE_SYSTEM,), + MESSAGE_TOOL: (MESSAGE_TOOL,), + CONTENT_TEXT: (CONTENT_TEXT,), + CONTENT_IMAGE: (CONTENT_IMAGE,), + CONTENT_MODEL_END_SAMPLING: (CONTENT_MODEL_END_SAMPLING,), + CONTENT_THINKING: (CONTENT_THINKING,), + CONTENT_AUDIO_INPUT: (CONTENT_AUDIO_INPUT,), + CONTENT_XML: (CONTENT_XML, "<|unused_200024|>"), + CONTENT_INVOKE_TOOL_JSON: (CONTENT_INVOKE_TOOL_JSON,), + END_MESSAGE: (END_MESSAGE,), + AUDIO_END: (AUDIO_END,), +} + + +class InklingTextTokenizer(Protocol): + """Structural tokenizer contract required by the renderer.""" + + def encode_text(self, text: str) -> list[int]: ... + + def encode_special(self, token: str) -> int: ... + + +# OpenAI content-part type spellings that mean image / audio (rendering only +# needs the kind, not the bytes — the bytes are handled by the MM processor). +_IMAGE_PART_TYPES = frozenset({"image", "input_image", "image_url"}) +_AUDIO_PART_TYPES = frozenset({"audio", "input_audio", "audio_url"}) + +_MAX_REASONING_EFFORT = 0.99 + + +def render_inkling_messages( + messages: Sequence[Mapping[str, Any]], + tokenizer: InklingTextTokenizer, + *, + add_generation_prompt: bool = True, + tools: Sequence[Mapping[str, Any]] | None = None, + reasoning_effort: float | None = None, +) -> list[int]: + """Render chat messages to Inkling input ids. + + PURE renderer: emits Inkling framing plus bare media markers; media + encoding and placeholder expansion happen later in the MM processor. + ``add_generation_prompt`` appends the assistant turn opener so the + model continues into the response. + """ + input_ids: list[int] = [] + tool_call_id_to_name: dict[str, str] = {} + + # Request-level tools plus per-developer-message tools (Rust renderer + # semantics) are declared in a single leading system block. + all_tools = list(tools or []) + for message in messages: + if message.get("role") == "developer": + all_tools.extend(message.get("tools") or []) + + if all_tools: + _append_message( + input_ids, + tokenizer, + "system", + "xml", + _tool_declare_json(all_tools), + author_name="tool_declare", + ) + + for message in messages: + role = _expect_role(message) + if role not in {"system", "developer"} and reasoning_effort is not None: + _append_reasoning_effort(input_ids, tokenizer, reasoning_effort) + reasoning_effort = None + + if role == "tool": + tool_name = message.get("name") or tool_call_id_to_name.get( + str(message.get("tool_call_id") or ""), "" + ) + _append_message( + input_ids, + tokenizer, + "tool", + "text", + _flatten_text_content(message.get("content")), + author_name=str(tool_name), + ) + continue + + if role == "assistant": + reasoning_content = message.get("reasoning") + if reasoning_content is None: + reasoning_content = message.get("reasoning_content") + if reasoning_content: + if not isinstance(reasoning_content, str): + raise TypeError( + "assistant reasoning_content must be a string for " + "Inkling rendering" + ) + _append_message( + input_ids, + tokenizer, + "assistant", + "thinking", + reasoning_content, + ) + + for kind, text in _iter_render_parts(message.get("content")): + _append_message(input_ids, tokenizer, role, kind, text) + + if role == "assistant": + for tool_call in message.get("tool_calls") or []: + name, args = _tool_call_name_and_args(tool_call) + tool_call_id = _as_mapping(tool_call).get("id") + if tool_call_id: + tool_call_id_to_name[str(tool_call_id)] = name + _append_message( + input_ids, + tokenizer, + "assistant", + "invoke_tool_json", + _tool_call_json(name, args), + author_name=name, + ) + + input_ids.append(tokenizer.encode_special(CONTENT_MODEL_END_SAMPLING)) + + if reasoning_effort is not None: + _append_reasoning_effort(input_ids, tokenizer, reasoning_effort) + + if add_generation_prompt: + input_ids.append(tokenizer.encode_special(MESSAGE_MODEL)) + return input_ids + + +def _append_reasoning_effort( + input_ids: list[int], + tokenizer: InklingTextTokenizer, + reasoning_effort: float, +) -> None: + _append_message( + input_ids, + tokenizer, + "system", + "text", + _thinking_effort_text(reasoning_effort), + ) + + +def _thinking_effort_text(reasoning_effort: Any) -> str: + if isinstance(reasoning_effort, bool) or not isinstance( + reasoning_effort, (int, float) + ): + raise TypeError( + "Inkling reasoning_effort must be a number in [0.0, 0.99], " + f"got {type(reasoning_effort).__name__}" + ) + value = float(reasoning_effort) + if not 0.0 <= value <= _MAX_REASONING_EFFORT: + raise ValueError( + f"Inkling reasoning_effort must be in [0.0, 0.99], got {value}" + ) + + effort_text = f"{value:.2f}".rstrip("0").rstrip(".") + if effort_text in {"0", "-0"}: + effort_text = "0.0" + return f"Thinking effort level: {effort_text}" + + +def _append_message( + input_ids: list[int], + tokenizer: InklingTextTokenizer, + role: str, + kind: str, + text: str, + *, + author_name: str | None = None, +) -> None: + input_ids.append(tokenizer.encode_special(ROLE_MESSAGE_TOKENS[role])) + if author_name: + input_ids.extend(tokenizer.encode_text(author_name)) + + if kind == "text": + input_ids.append(tokenizer.encode_special(CONTENT_TEXT)) + input_ids.extend(tokenizer.encode_text(text)) + elif kind == "image": + # Bare marker only: the MM processor replaces the marker with + # `marker + placeholder * num_patches` (see mm_preprocess.py). + input_ids.append(tokenizer.encode_special(CONTENT_IMAGE)) + elif kind == "audio": + input_ids.append(tokenizer.encode_special(CONTENT_AUDIO_INPUT)) + input_ids.append(tokenizer.encode_special(AUDIO_END)) + elif kind == "thinking": + input_ids.append(tokenizer.encode_special(CONTENT_THINKING)) + input_ids.extend(tokenizer.encode_text(text)) + elif kind == "xml": + input_ids.append(tokenizer.encode_special(CONTENT_XML)) + input_ids.extend(tokenizer.encode_text(text)) + elif kind == "invoke_tool_json": + input_ids.append(tokenizer.encode_special(CONTENT_INVOKE_TOOL_JSON)) + input_ids.extend(tokenizer.encode_text(text)) + else: + raise ValueError(f"unsupported Inkling render part kind: {kind!r}") + + input_ids.append(tokenizer.encode_special(END_MESSAGE)) + + +def _iter_render_parts(content: Any) -> Iterator[tuple[str, str]]: + """Yield (kind, text) per content part: kind in {text, image, audio}.""" + if content is None: + return + if isinstance(content, str): + if content: + yield ("text", content) + return + if not isinstance(content, Sequence) or isinstance(content, (bytes, bytearray)): + raise TypeError("message content must be a string or a sequence of parts") + for part in content: + if isinstance(part, str): + yield ("text", part) + continue + if not isinstance(part, Mapping): + raise TypeError(f"content part must be mapping, got {type(part).__name__}") + ptype = part.get("type") + if ptype in (None, "text", "input_text"): + text = part.get("text", "") + yield ("text", text if isinstance(text, str) else "") + elif ptype in _IMAGE_PART_TYPES: + yield ("image", "") + elif ptype in _AUDIO_PART_TYPES: + yield ("audio", "") + else: + raise ValueError(f"unsupported content part type: {ptype!r}") + + +def _flatten_text_content(content: Any) -> str: + """Flatten a tool-response content (string or text parts) to text.""" + if content is None: + return "" + if isinstance(content, str): + return content + parts: list[str] = [] + for kind, text in _iter_render_parts(content): + if kind != "text": + raise ValueError( + "Inkling tool response content must be text, " + f"got a part of kind {kind!r}" + ) + parts.append(text) + return "".join(parts) + + +def _expect_role(message: Mapping[str, Any]) -> str: + role = message.get("role") + if role not in ROLE_MESSAGE_TOKENS: + raise ValueError( + f"unsupported Inkling message role {role!r}; " + f"expected one of {sorted(ROLE_MESSAGE_TOKENS)}" + ) + return str(role) + + +def _as_mapping(value: Any) -> Mapping[str, Any]: + if isinstance(value, Mapping): + return value + if hasattr(value, "model_dump"): + dumped = value.model_dump() + if isinstance(dumped, Mapping): + return dumped + raise TypeError(f"expected mapping, got {type(value).__name__}") + + +def _canonical_json(value: Any) -> str: + return json.dumps( + _sort_json(value), + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + ) + + +def _sort_json(value: Any) -> Any: + if isinstance(value, Mapping): + return {str(key): _sort_json(value[key]) for key in sorted(value)} + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return [_sort_json(item) for item in value] + return value + + +def _tool_declare_json(tools: Sequence[Mapping[str, Any]]) -> str: + tool_specs = [] + for tool_value in tools: + tool = _as_mapping(tool_value) + function = _as_mapping(tool.get("function", {})) + tool_specs.append( + { + "description": function.get("description") or "", + "name": function["name"], + "parameters": function.get("parameters") or {}, + "type": tool.get("type", "function"), + } + ) + return _canonical_json(tool_specs) + + +def _tool_call_name_and_args(tool_call_value: Any) -> tuple[str, Mapping[str, Any]]: + tool_call = _as_mapping(tool_call_value) + function = _as_mapping(tool_call.get("function", {})) + name = function.get("name") + if not isinstance(name, str): + raise TypeError("tool call function name must be a string") + + raw_args = function.get("arguments") or {} + if isinstance(raw_args, str): + args = json.loads(raw_args) if raw_args.strip() else {} + else: + args = raw_args + if not isinstance(args, Mapping): + raise TypeError("tool call function arguments must decode to an object") + return name, args + + +def _tool_call_json(name: str, args: Mapping[str, Any]) -> str: + name_json = json.dumps(name, ensure_ascii=False, allow_nan=False) + return f'{{"name":{name_json},"args":{_canonical_json(args)}}}' diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index 098a58e8ed..395132c56e 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -26,6 +26,7 @@ _VLLM_RENDERERS = { "kimi_audio": ("hf", "HfRenderer"), "mistral": ("mistral", "MistralRenderer"), "terratorch": ("terratorch", "TerratorchRenderer"), + "inkling": ("inkling", "InklingRenderer"), } diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index cef2f7645f..e6c12ccc3b 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -44,6 +44,10 @@ _VLLM_TOKENIZERS = { "hf": ("hf", "CachedHfTokenizer"), "kimi_audio": ("kimi_audio", "KimiAudioTokenizer"), "mistral": ("mistral", "MistralTokenizer"), + # Inkling uses the plain HF tokenizer for token operations; the "inkling" + # mode exists to select the InklingRenderer, which renders chat to + # token ids natively (Inkling has no Jinja chat template). + "inkling": ("hf", "CachedHfTokenizer"), } diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index 26362ebf0e..64b1342fd2 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -174,6 +174,10 @@ _TOOL_PARSERS_TO_REGISTER = { "step3p5_tool_parser", "Step3p5ToolParser", ), + "inkling": ( + "inkling_tool_parser", + "InklingEngineToolParser", + ), "xlam": ( "xlam_tool_parser", "xLAMToolParser", diff --git a/vllm/tool_parsers/inkling_tool_parser.py b/vllm/tool_parsers/inkling_tool_parser.py new file mode 100644 index 0000000000..ab760015af --- /dev/null +++ b/vllm/tool_parsers/inkling_tool_parser.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import InklingParserToolAdapter + + +class InklingEngineToolParser(InklingParserToolAdapter): # type: ignore[valid-type, misc] + # No Inkling structural-tag grammar is wired up yet; fall back to auto + # parsing for named/required tool choice. + structural_tag_model = None + supports_required_and_named = False diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index bb72087a95..b457a6e2f0 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -127,6 +127,8 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( laguna="LagunaConfig", lfm2_moe="Lfm2MoeConfig", **{"unlimited-ocr": "UnlimitedOCRConfig"}, + inkling_mm_model="InklingMMConfig", + inkling_model="InklingModelConfig", ) _SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators", "medusa"} diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index b64310507b..4bb7674ddb 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -94,6 +94,10 @@ _CLASS_TO_MODULE: dict[str, str] = { "Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5", "Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe", "Qwen3_5MoeTextConfig": "vllm.transformers_utils.configs.qwen3_5_moe", + "InklingModelConfig": "vllm.models.inkling.configs", + "InklingAudioConfig": "vllm.models.inkling.configs", + "InklingVisionConfig": "vllm.models.inkling.configs", + "InklingMMConfig": "vllm.models.inkling.configs", # Special case: DeepseekV3Config is from HuggingFace Transformers "DeepseekV3Config": "transformers", } @@ -174,6 +178,10 @@ __all__ = [ "Qwen3_5TextConfig", "Qwen3_5MoeConfig", "Qwen3_5MoeTextConfig", + "InklingModelConfig", + "InklingAudioConfig", + "InklingVisionConfig", + "InklingMMConfig", ] diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py index e35218c4be..60b56b75a0 100644 --- a/vllm/transformers_utils/processors/__init__.py +++ b/vllm/transformers_utils/processors/__init__.py @@ -44,6 +44,9 @@ __all__ = [ "Ovis2_5Processor", "Qwen3ASRProcessor", "Step3VLProcessor", + "InklingProcessor", + "InklingImageProcessor", + "InklingAudioFeatureExtractor", ] _CLASS_TO_MODULE: dict[str, str] = { @@ -80,6 +83,9 @@ _CLASS_TO_MODULE: dict[str, str] = { "Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5", "Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr", "Step3VLProcessor": "vllm.transformers_utils.processors.step3_vl", + "InklingProcessor": "vllm.transformers_utils.processors.inkling", + "InklingImageProcessor": "vllm.transformers_utils.processors.inkling", + "InklingAudioFeatureExtractor": "vllm.transformers_utils.processors.inkling", } diff --git a/vllm/transformers_utils/processors/inkling.py b/vllm/transformers_utils/processors/inkling.py new file mode 100644 index 0000000000..09d34f7806 --- /dev/null +++ b/vllm/transformers_utils/processors/inkling.py @@ -0,0 +1,504 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Vendored HuggingFace-convention processors for the Inkling Titan model. + +Implements the image processor (numba patchifier), the audio feature extractor +(STFT/dMel path), the composite processor, and the MM token-id constants. + +Besides raw bytes / file paths, the extractors also accept the dummy inputs +vLLM generates during profiling (PIL images / numpy audio arrays). +""" + +from __future__ import annotations + +import io +import math +from collections.abc import Sequence +from dataclasses import dataclass + +import numpy as np +import torch +import torch.nn.functional as F +from numba import njit +from transformers.feature_extraction_utils import ( + BatchFeature, + FeatureExtractionMixin, +) +from transformers.image_processing_utils import BaseImageProcessor +from transformers.image_utils import ImageInput + +# --------------------------------------------------------------------------- +# MM token-id constants +# --------------------------------------------------------------------------- + +# Block-start marker token ids. These are real tokens the model was trained on +# (``<|content_image|>`` / ``<|content_audio_input|>``) that mark the start of an +# image/audio embedding block; they are kept verbatim in ``input_ids``. +IMAGE_MARKER_ID = 200005 # <|content_image|> +AUDIO_MARKER_ID = 200020 # <|content_audio_input|> + +# Per-patch / per-frame placeholder ids marking where tower embeddings are +# scattered in. These are unused slots in the padded vocabulary and the +# corresponding positions are always overwritten by tower embeddings. +IMAGE_TOKEN_ID = 200054 # <|unused_200054|> +AUDIO_TOKEN_ID = 200053 # <|unused_200053|> + +# --------------------------------------------------------------------------- +# Image processing +# --------------------------------------------------------------------------- + +IMAGE_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32) +IMAGE_STD = np.array([0.26862954, 0.2613026, 0.2757771], dtype=np.float32) +PAD_RAW_VALUE = np.float32(-1.0 / 255.0) +PAD_NORM = (np.full((3,), PAD_RAW_VALUE, dtype=np.float32) - IMAGE_MEAN) / IMAGE_STD + + +def _validate_image_rescale( + rescale_image_frac: float | None, + rescale_image_max_upscaled_long_edge: int | None, +) -> None: + if rescale_image_frac is not None and ( + not math.isfinite(rescale_image_frac) or rescale_image_frac <= 0 + ): + raise ValueError( + "rescale_image_frac must be positive and finite or None, " + f"got {rescale_image_frac}" + ) + if rescale_image_max_upscaled_long_edge is None: + return + if rescale_image_max_upscaled_long_edge <= 0: + raise ValueError( + "rescale_image_max_upscaled_long_edge must be positive or None, " + f"got {rescale_image_max_upscaled_long_edge}" + ) + if rescale_image_frac is None or rescale_image_frac <= 1.0: + raise ValueError( + "rescale_image_max_upscaled_long_edge requires rescale_image_frac > 1, " + f"got {rescale_image_frac}" + ) + + +def _scaled_image_dimensions( + width: int, + height: int, + rescale_image_frac: float | None, + rescale_image_max_upscaled_long_edge: int | None, +) -> tuple[int, int]: + """Return the long-edge-scaled ``(width, height)``.""" + if rescale_image_frac is None: + return width, height + + long_edge = max(width, height) + if long_edge == 0: + return width, height + + target_long_edge = float(long_edge) * rescale_image_frac + if rescale_image_max_upscaled_long_edge is not None: + effective_cap = max(rescale_image_max_upscaled_long_edge, long_edge) + target_long_edge = min(target_long_edge, float(effective_cap)) + + ratio = target_long_edge / float(long_edge) + if ratio == 1.0: + return width, height + + def scale(value: int) -> int: + return max(1, math.floor(float(value) * ratio + 0.5)) + + return scale(width), scale(height) + + +def _load_image_bytes(image) -> bytes: + """Encode a PIL image as raw PNG bytes for preprocessing. + + The HF processor is always handed ``PIL.Image`` instances by vLLM, so no + other input types need to be supported here. + """ + if image.mode != "RGB": + image = image.convert("RGB") + buf = io.BytesIO() + image.save(buf, format="PNG") + return buf.getvalue() + + +@njit(cache=True) +def _fill_patches_numba( + arr: np.ndarray, + patch_size: int, + patches: np.ndarray, + mean: np.ndarray, + std: np.ndarray, + pad_norm: np.ndarray, +) -> None: + h = arr.shape[0] + w = arr.shape[1] + nph = (h + patch_size - 1) // patch_size + npw = w // patch_size + 1 + inv255 = np.float32(1.0 / 255.0) + + for k in range(nph * npw): + i = k // npw + j = k - i * npw + y_base = i * patch_size + x_base = j * patch_size + + for y in range(patch_size): + iy = y_base + y + for x in range(patch_size): + ix = x_base + x + if iy < h and ix < w: + for c in range(3): + raw = np.float32(arr[iy, ix, c]) * inv255 + patches[k, y, x, c] = (raw - mean[c]) / std[c] + else: + for c in range(3): + patches[k, y, x, c] = pad_norm[c] + + +def _encode_image_bytes( + image_bytes: bytes, + *, + patch_size: int, + rescale_image_frac: float | None, + rescale_image_max_upscaled_long_edge: int | None, +) -> torch.Tensor: + if patch_size <= 0: + raise ValueError("patch_size must be greater than zero") + _validate_image_rescale( + rescale_image_frac, + rescale_image_max_upscaled_long_edge, + ) + + from PIL import Image + + image = Image.open(io.BytesIO(image_bytes)).convert("RGB") + scaled_size = _scaled_image_dimensions( + image.width, + image.height, + rescale_image_frac=rescale_image_frac, + rescale_image_max_upscaled_long_edge=rescale_image_max_upscaled_long_edge, + ) + if scaled_size != image.size: + image = image.resize(scaled_size, resample=Image.Resampling.LANCZOS) + arr = np.array(image, dtype=np.uint8, copy=True) + height, width, _ = arr.shape + + nph = (height + patch_size - 1) // patch_size + npw = width // patch_size + 1 + num_patches = nph * npw + + patches = np.empty((num_patches, patch_size, patch_size, 3), dtype=np.float32) + _fill_patches_numba(arr, patch_size, patches, IMAGE_MEAN, IMAGE_STD, PAD_NORM) + + return ( + torch.from_numpy(patches) + .to(torch.bfloat16) + .view(num_patches, 1, patch_size, patch_size, 3) + .expand(num_patches, 2, patch_size, patch_size, 3) + ) + + +class InklingImageProcessor(BaseImageProcessor): + r"""Turn raw images into ``vision_patches_bthwc`` for Inkling hMLP. + + ``rescale_image_frac`` scales the long edge while preserving aspect ratio. + ``rescale_image_max_upscaled_long_edge`` optionally caps only upscaling and + therefore requires a scale factor greater than one. The defaults, ``2.0`` and + ``2048``, grow images toward a 2048-pixel long edge by at most 2x, while leaving + images already at or above 2048 unchanged. + """ + + model_input_names = ["vision_patches_bthwc"] + + def __init__( + self, + patch_size: int = 40, + rescale_image_frac: float | None = 2.0, + rescale_image_max_upscaled_long_edge: int | None = 2048, + **kwargs, + ): + if patch_size <= 0: + raise ValueError("patch_size must be greater than zero") + _validate_image_rescale( + rescale_image_frac, + rescale_image_max_upscaled_long_edge, + ) + super().__init__(**kwargs) + self.patch_size = patch_size + self.rescale_image_frac = rescale_image_frac + self.rescale_image_max_upscaled_long_edge = rescale_image_max_upscaled_long_edge + + def _encode_one(self, image) -> torch.Tensor: + return _encode_image_bytes( + _load_image_bytes(image), + patch_size=self.patch_size, + rescale_image_frac=self.rescale_image_frac, + rescale_image_max_upscaled_long_edge=self.rescale_image_max_upscaled_long_edge, + ) + + def preprocess( + self, + images: ImageInput | list, + return_tensors: str | None = "pt", + **kwargs, + ) -> BatchFeature: + del return_tensors, kwargs + if not isinstance(images, (list, tuple)): + images = [images] + + per_image_patches: list[torch.Tensor] = [] + num_patches: list[int] = [] + num_tokens: list[int] = [] + for img in images: + vp = self._encode_one(img) + n_patches = int(vp.shape[0]) + per_image_patches.append(vp) + num_patches.append(n_patches) + num_tokens.append(n_patches) + + if len(per_image_patches) == 1: + vision_patches_bthwc = per_image_patches[0] + elif per_image_patches: + vision_patches_bthwc = torch.cat(per_image_patches, dim=0) + else: + vision_patches_bthwc = torch.empty(0) + + data = { + "vision_patches_bthwc": vision_patches_bthwc, + "num_patches": num_patches, + "num_tokens": num_tokens, + } + return BatchFeature(data=data, tensor_type=None) + + +# --------------------------------------------------------------------------- +# Audio feature extraction +# --------------------------------------------------------------------------- + + +@dataclass +class InklingAudioEncoderParams: + """Audio preprocessing parameters used to convert raw audio into dMel bins.""" + + sample_rate: int = 16_000 + window_size_multiplier: float = 2.0 + n_fft: int | None = None + n_mels: int = 80 + num_dmel_bins: int = 16 + dmel_min_value: float = -7.0 + dmel_max_value: float = 2.0 + audio_token_duration_s: float = 0.05 + + +def _to_exact_int(value: float, name: str, tolerance: float = 1e-6) -> int: + rounded = round(value) + if abs(value - rounded) > tolerance: + raise ValueError(f"{name} must resolve to an integer sample count, got {value}") + return int(rounded) + + +def _hz_to_mel(frequencies: np.ndarray) -> np.ndarray: + """Slaney mel scale, matching the librosa/torchaudio convention.""" + frequencies = np.asarray(frequencies, dtype=np.float64) + f_sp = 200.0 / 3.0 + min_log_hz = 1000.0 + min_log_mel = min_log_hz / f_sp + logstep = np.log(6.4) / 27.0 + linear = frequencies / f_sp + log = ( + min_log_mel + np.log(np.maximum(frequencies, min_log_hz) / min_log_hz) / logstep + ) + return np.where(frequencies >= min_log_hz, log, linear) + + +def _mel_to_hz(mels: np.ndarray) -> np.ndarray: + mels = np.asarray(mels, dtype=np.float64) + f_sp = 200.0 / 3.0 + min_log_hz = 1000.0 + min_log_mel = min_log_hz / f_sp + logstep = np.log(6.4) / 27.0 + linear = mels * f_sp + log = min_log_hz * np.exp(logstep * (mels - min_log_mel)) + return np.where(mels >= min_log_mel, log, linear) + + +_MEL_BASIS_CACHE: dict[tuple[int, int, int], torch.Tensor] = {} + + +def _mel_basis(sample_rate: int, n_fft: int, n_mels: int) -> torch.Tensor: + key = (sample_rate, n_fft, n_mels) + cached = _MEL_BASIS_CACHE.get(key) + if cached is not None: + return cached + + fft_bins = n_fft // 2 + 1 + fft_freqs = np.arange(fft_bins, dtype=np.float64) * sample_rate / n_fft + mel_edges = _mel_to_hz( + np.linspace( + _hz_to_mel(np.array([0.0]))[0], + _hz_to_mel(np.array([sample_rate / 2.0]))[0], + n_mels + 2, + dtype=np.float64, + ) + ) + mel_widths = np.diff(mel_edges) + lower = (fft_freqs[None, :] - mel_edges[:-2, None]) / mel_widths[:-1, None] + upper = (mel_edges[2:, None] - fft_freqs[None, :]) / mel_widths[1:, None] + weights = np.maximum(0.0, np.minimum(lower, upper)) + + # Slaney area normalization. + weights *= (2.0 / (mel_edges[2:] - mel_edges[:-2]))[:, None] + basis = torch.from_numpy(weights.astype(np.float32, copy=False)).contiguous() + _MEL_BASIS_CACHE[key] = basis + return basis + + +def _dmel_bins(audio: torch.Tensor, params: InklingAudioEncoderParams) -> torch.Tensor: + hop_length = _to_exact_int( + params.audio_token_duration_s * params.sample_rate, + "audio_token_duration_s * sample_rate", + ) + window_size = _to_exact_int( + params.audio_token_duration_s + * params.window_size_multiplier + * params.sample_rate, + "audio_token_duration_s * window_size_multiplier * sample_rate", + ) + n_fft = params.n_fft or window_size + if hop_length <= 0 or window_size <= 0 or n_fft <= 0: + raise ValueError("audio hop length, window size, and n_fft must be positive") + if audio.numel() == 0: + return torch.empty((0, params.n_mels), dtype=torch.int32) + + right_pad = math.ceil(audio.numel() / hop_length) * hop_length - audio.numel() + left_pad = max(n_fft - hop_length, 0) + audio = F.pad(audio, (left_pad, right_pad)) + + window = torch.hann_window(window_size, periodic=True, dtype=torch.float32) + spec = torch.stft( + audio.unsqueeze(0), + n_fft=n_fft, + hop_length=hop_length, + win_length=window_size, + window=window, + center=False, + normalized=False, + onesided=True, + return_complex=True, + ) + spec_ri = torch.view_as_real(spec) + magnitude = ( + (spec_ri[..., 0].square() + spec_ri[..., 1].square()) + .clamp_min(1e-10) + .sqrt() + .squeeze(0) + ) + + mel = ( + _mel_basis(params.sample_rate, n_fft, params.n_mels) + .matmul(magnitude) + .clamp_min(1e-10) + .log10() + ) + mel = mel.to(torch.float64).clamp( + min=params.dmel_min_value, max=params.dmel_max_value + ) + bin_centers = torch.linspace( + params.dmel_min_value, + params.dmel_max_value, + params.num_dmel_bins, + dtype=torch.float64, + ) + dmel_bins = (mel.unsqueeze(-1) - bin_centers).abs().argmin(dim=-1) + return dmel_bins.to(torch.int32).T.contiguous() + + +class InklingAudioFeatureExtractor(FeatureExtractionMixin): + """Convert raw audio into Inkling dMel bins in the HF feature-extractor API.""" + + model_input_names = ["dmel_bins"] + + def __init__(self, params: dict | None = None, **kwargs): + super().__init__(**kwargs) + merged = InklingAudioEncoderParams() + if params: + for k, v in params.items(): + if hasattr(merged, k): + setattr(merged, k, v) + # also accept flat kwargs (HF config style) + for k in list(kwargs.keys()): + if hasattr(merged, k): + setattr(merged, k, kwargs[k]) + self.params = merged + + def _decode_one(self, audio) -> torch.Tensor: + # vLLM hands the feature extractor numpy arrays (the dummy-input builder + # during profiling, and MultiModalDataParser after resampling to the + # target sample rate), so no other input types need to be supported. + return torch.from_numpy( + np.ascontiguousarray(audio.astype(np.float32, copy=False)) + ).flatten() + + def _encode_one(self, audio) -> torch.Tensor: + return _dmel_bins(self._decode_one(audio), self.params) + + def __call__( + self, + audios: Sequence | None, + return_tensors: str | None = None, + **kwargs, + ) -> BatchFeature: + del return_tensors, kwargs + if audios is None: + audios = [] + if not isinstance(audios, (list, tuple)): + audios = [audios] + + dmel_bins = [self._encode_one(a) for a in audios] + data = { + # per-clip feature: dmel bins as float32 [T, n_mels] + "dmel_bins": [bins.to(torch.float32) for bins in dmel_bins], + "num_audio_tokens": [int(bins.shape[0]) for bins in dmel_bins], + } + # return_tensors intentionally ignored: per-clip features have ragged T. + return BatchFeature(data=data, tensor_type=None) + + +# --------------------------------------------------------------------------- +# Composite processor +# --------------------------------------------------------------------------- + + +class InklingProcessor: + """Bundle Inkling image + audio preprocessing with the MM token ids.""" + + def __init__( + self, + image_processor: InklingImageProcessor | None = None, + audio_feature_extractor: InklingAudioFeatureExtractor | None = None, + tokenizer=None, + ): + self.image_processor = image_processor or InklingImageProcessor() + self.audio_feature_extractor = ( + audio_feature_extractor or InklingAudioFeatureExtractor() + ) + self.tokenizer = tokenizer + + def process_images(self, images: list): + """Raw images -> BatchFeature(vision_patches_bthwc, num_patches, num_tokens).""" + return self.image_processor.preprocess(images, return_tensors="pt") + + def process_audios(self, audios: list): + """Raw audios -> BatchFeature(dmel_bins, num_audio_tokens).""" + return self.audio_feature_extractor(audios) + + +__all__ = [ + "InklingImageProcessor", + "InklingAudioFeatureExtractor", + "InklingAudioEncoderParams", + "InklingProcessor", + "IMAGE_MARKER_ID", + "AUDIO_MARKER_ID", + "IMAGE_TOKEN_ID", + "AUDIO_TOKEN_ID", +]