[Model] Add Inkling model support [1/N] (#48799)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Woosuk Kwon
2026-07-15 23:40:07 -07:00
committed by GitHub
co-authored by Bugen Zhao Giancarlo Delfin Isotr0py Isotr0py Jee Jee Li Roger Wang Yifan Qiao Claude Fable 5 OpenAI Codex
parent 8bfd683901
commit 6570c9800c
95 changed files with 12137 additions and 62 deletions
+46 -3
View File
@@ -95,6 +95,7 @@ impl HfSpecialTokens {
pub struct ModelConfig {
model_type: Option<String>,
vocab_size: Option<u32>,
eos_token_id: Option<OneOrManyTokenIds>,
num_experts: Option<OneOrManyExpertCount>,
moe_num_experts: Option<OneOrManyExpertCount>,
n_routed_experts: Option<OneOrManyExpertCount>,
@@ -125,12 +126,16 @@ pub(super) enum OneOrManyTokenIds {
}
impl OneOrManyTokenIds {
pub(super) fn into_set(self) -> BTreeSet<u32> {
pub(super) fn as_slice(&self) -> &[u32] {
match self {
Self::One(id) => BTreeSet::from([id]),
Self::Many(ids) => ids.into_iter().collect(),
Self::One(id) => std::slice::from_ref(id),
Self::Many(ids) => ids.as_slice(),
}
}
pub(super) fn into_set(self) -> BTreeSet<u32> {
self.as_slice().iter().copied().collect()
}
}
/// Hugging Face configs may expose the expert count either as one integer or
@@ -195,6 +200,18 @@ impl ModelConfig {
}
}
/// Return the effective model-side EOS token ids, following the same
/// simplified text-config selection as `vocab_size`.
pub(super) fn eos_token_ids(&self) -> &[u32] {
if let Some(eos_token_id) = self.eos_token_id.as_ref() {
eos_token_id.as_slice()
} else if let Some(text_config) = self.text_config.as_deref() {
text_config.eos_token_ids()
} else {
&[]
}
}
/// Match Python's current expert-count priority on the selected text
/// config.
///
@@ -354,6 +371,32 @@ mod tests {
assert_eq!(config.vocab_size().unwrap(), 151936);
}
#[test]
fn model_config_reads_top_level_eos_token_ids() {
let single: ModelConfig = serde_json::from_str(r#"{"eos_token_id":151645}"#).unwrap();
let many: ModelConfig =
serde_json::from_str(r#"{"eos_token_id":[128001,128008,128009]}"#).unwrap();
let null: ModelConfig = serde_json::from_str(r#"{"eos_token_id":null}"#).unwrap();
assert_eq!(single.eos_token_ids(), &[151645]);
assert_eq!(many.eos_token_ids(), &[128001, 128008, 128009]);
assert!(null.eos_token_ids().is_empty());
}
#[test]
fn model_config_uses_nested_eos_token_ids_when_top_level_is_absent() {
let config: ModelConfig = serde_json::from_str(
r#"{
"text_config": {
"eos_token_id": [59246, 59253, 59255]
}
}"#,
)
.unwrap();
assert_eq!(config.eos_token_ids(), &[59246, 59253, 59255]);
}
#[test]
fn model_config_rejects_missing_vocab_size() {
let config: ModelConfig = serde_json::from_str(r#"{}"#).unwrap();
+119 -15
View File
@@ -8,7 +8,9 @@ use std::collections::BTreeSet;
use std::sync::Arc;
use tracing::info;
use vllm_tokenizer::{DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer};
use vllm_tokenizer::{
DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer, Tokenizer,
};
use self::config::{GenerationConfig, load_generation_config};
pub use self::config::{
@@ -56,23 +58,15 @@ impl HfTextBackend {
pub fn from_resolved_model_files(files: ResolvedModelFiles, model_id: String) -> Result<Self> {
let tokenizer_config = load_tokenizer_config(files.tokenizer_config_path.as_deref())?;
let tokenizer = load_tokenizer(&files.tokenizer)?;
let primary_eos_token_id = tokenizer_config
.special_tokens
.eos_token
.as_ref()
.and_then(|token| tokenizer.token_to_id(token.as_str()));
let model_config = load_model_config(files.config_path.as_deref())?;
let model_vocab_size = model_config.vocab_size()? as usize;
let generation_config = load_generation_config(files.generation_config_path.as_deref())?;
let mut extra_eos_token_ids = generation_config
.eos_token_id
.clone()
.map(|value| value.into_set())
.unwrap_or_default();
if let Some(primary_eos_token_id) = primary_eos_token_id {
extra_eos_token_ids.remove(&primary_eos_token_id);
}
let (primary_eos_token_id, extra_eos_token_ids) = resolve_eos_token_ids(
&tokenizer_config,
&model_config,
&generation_config,
tokenizer.as_ref(),
);
info!(
model_id,
@@ -98,6 +92,42 @@ impl HfTextBackend {
}
}
/// Resolve EOS hints from tokenizer, model, and generation configs.
///
/// Resolution rules:
/// 1. Use the tokenizer-side EOS token as the primary EOS when it resolves to a
/// token id.
/// 2. Fall back to the first model-config EOS id when the tokenizer config does
/// not provide a primary EOS.
/// 3. Keep any remaining model/generation EOS ids as extra stop-token ids so
/// they still participate in stopping and min-token handling.
fn resolve_eos_token_ids(
tokenizer_config: &HfTokenizerConfig,
model_config: &ModelConfig,
generation_config: &GenerationConfig,
tokenizer: &dyn Tokenizer,
) -> (Option<u32>, BTreeSet<u32>) {
let model_config_eos_token_ids = model_config.eos_token_ids();
let primary_eos_token_id = tokenizer_config
.special_tokens
.eos_token
.as_ref()
.and_then(|token| tokenizer.token_to_id(token.as_str()))
.or_else(|| model_config_eos_token_ids.first().copied());
let mut extra_eos_token_ids = generation_config
.eos_token_id
.clone()
.map(|value| value.into_set())
.unwrap_or_default();
extra_eos_token_ids.extend(model_config_eos_token_ids.iter().copied());
if let Some(primary_eos_token_id) = primary_eos_token_id {
extra_eos_token_ids.remove(&primary_eos_token_id);
}
(primary_eos_token_id, extra_eos_token_ids)
}
impl TextBackend for HfTextBackend {
fn tokenizer(&self) -> DynTokenizer {
self.tokenizer.clone()
@@ -128,3 +158,77 @@ impl TextBackend for HfTextBackend {
})
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use super::{GenerationConfig, HfTokenizerConfig, ModelConfig, resolve_eos_token_ids};
use vllm_tokenizer::Tokenizer;
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(vec![])
}
fn decode(
&self,
_token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(String::new())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
(token == "</s>").then_some(2)
}
fn id_to_token(&self, id: u32) -> Option<String> {
(id == 2).then(|| "</s>".to_string())
}
}
#[test]
fn eos_resolution_uses_model_config_when_tokenizer_has_no_eos() {
let tokenizer_config: HfTokenizerConfig = serde_json::from_str("{}").unwrap();
let model_config: ModelConfig =
serde_json::from_str(r#"{"eos_token_id":[200006,200010]}"#).unwrap();
let generation_config: GenerationConfig = serde_json::from_str("{}").unwrap();
let (primary, extra) = resolve_eos_token_ids(
&tokenizer_config,
&model_config,
&generation_config,
&FakeTokenizer,
);
assert_eq!(primary, Some(200006));
assert_eq!(extra, BTreeSet::from([200010]));
}
#[test]
fn eos_resolution_keeps_tokenizer_eos_primary() {
let tokenizer_config: HfTokenizerConfig =
serde_json::from_str(r#"{"eos_token":"</s>"}"#).unwrap();
let model_config: ModelConfig =
serde_json::from_str(r#"{"eos_token_id":[2,200006]}"#).unwrap();
let generation_config: GenerationConfig =
serde_json::from_str(r#"{"eos_token_id":[2,200010]}"#).unwrap();
let (primary, extra) = resolve_eos_token_ids(
&tokenizer_config,
&model_config,
&generation_config,
&FakeTokenizer,
);
assert_eq!(primary, Some(2));
assert_eq!(extra, BTreeSet::from([200006, 200010]));
}
}