[Model] Add Inkling model support [1/N] (#48799)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Bugen Zhao
Giancarlo Delfin
Isotr0py
Isotr0py
Jee Jee Li
Roger Wang
Yifan Qiao
Claude Fable 5
OpenAI Codex
parent
8bfd683901
commit
6570c9800c
@@ -95,6 +95,7 @@ impl HfSpecialTokens {
|
||||
pub struct ModelConfig {
|
||||
model_type: Option<String>,
|
||||
vocab_size: Option<u32>,
|
||||
eos_token_id: Option<OneOrManyTokenIds>,
|
||||
num_experts: Option<OneOrManyExpertCount>,
|
||||
moe_num_experts: Option<OneOrManyExpertCount>,
|
||||
n_routed_experts: Option<OneOrManyExpertCount>,
|
||||
@@ -125,12 +126,16 @@ pub(super) enum OneOrManyTokenIds {
|
||||
}
|
||||
|
||||
impl OneOrManyTokenIds {
|
||||
pub(super) fn into_set(self) -> BTreeSet<u32> {
|
||||
pub(super) fn as_slice(&self) -> &[u32] {
|
||||
match self {
|
||||
Self::One(id) => BTreeSet::from([id]),
|
||||
Self::Many(ids) => ids.into_iter().collect(),
|
||||
Self::One(id) => std::slice::from_ref(id),
|
||||
Self::Many(ids) => ids.as_slice(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn into_set(self) -> BTreeSet<u32> {
|
||||
self.as_slice().iter().copied().collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Hugging Face configs may expose the expert count either as one integer or
|
||||
@@ -195,6 +200,18 @@ impl ModelConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the effective model-side EOS token ids, following the same
|
||||
/// simplified text-config selection as `vocab_size`.
|
||||
pub(super) fn eos_token_ids(&self) -> &[u32] {
|
||||
if let Some(eos_token_id) = self.eos_token_id.as_ref() {
|
||||
eos_token_id.as_slice()
|
||||
} else if let Some(text_config) = self.text_config.as_deref() {
|
||||
text_config.eos_token_ids()
|
||||
} else {
|
||||
&[]
|
||||
}
|
||||
}
|
||||
|
||||
/// Match Python's current expert-count priority on the selected text
|
||||
/// config.
|
||||
///
|
||||
@@ -354,6 +371,32 @@ mod tests {
|
||||
assert_eq!(config.vocab_size().unwrap(), 151936);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_reads_top_level_eos_token_ids() {
|
||||
let single: ModelConfig = serde_json::from_str(r#"{"eos_token_id":151645}"#).unwrap();
|
||||
let many: ModelConfig =
|
||||
serde_json::from_str(r#"{"eos_token_id":[128001,128008,128009]}"#).unwrap();
|
||||
let null: ModelConfig = serde_json::from_str(r#"{"eos_token_id":null}"#).unwrap();
|
||||
|
||||
assert_eq!(single.eos_token_ids(), &[151645]);
|
||||
assert_eq!(many.eos_token_ids(), &[128001, 128008, 128009]);
|
||||
assert!(null.eos_token_ids().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_uses_nested_eos_token_ids_when_top_level_is_absent() {
|
||||
let config: ModelConfig = serde_json::from_str(
|
||||
r#"{
|
||||
"text_config": {
|
||||
"eos_token_id": [59246, 59253, 59255]
|
||||
}
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(config.eos_token_ids(), &[59246, 59253, 59255]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn model_config_rejects_missing_vocab_size() {
|
||||
let config: ModelConfig = serde_json::from_str(r#"{}"#).unwrap();
|
||||
|
||||
@@ -8,7 +8,9 @@ use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use tracing::info;
|
||||
use vllm_tokenizer::{DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer};
|
||||
use vllm_tokenizer::{
|
||||
DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer, Tokenizer,
|
||||
};
|
||||
|
||||
use self::config::{GenerationConfig, load_generation_config};
|
||||
pub use self::config::{
|
||||
@@ -56,23 +58,15 @@ impl HfTextBackend {
|
||||
pub fn from_resolved_model_files(files: ResolvedModelFiles, model_id: String) -> Result<Self> {
|
||||
let tokenizer_config = load_tokenizer_config(files.tokenizer_config_path.as_deref())?;
|
||||
let tokenizer = load_tokenizer(&files.tokenizer)?;
|
||||
let primary_eos_token_id = tokenizer_config
|
||||
.special_tokens
|
||||
.eos_token
|
||||
.as_ref()
|
||||
.and_then(|token| tokenizer.token_to_id(token.as_str()));
|
||||
|
||||
let model_config = load_model_config(files.config_path.as_deref())?;
|
||||
let model_vocab_size = model_config.vocab_size()? as usize;
|
||||
let generation_config = load_generation_config(files.generation_config_path.as_deref())?;
|
||||
let mut extra_eos_token_ids = generation_config
|
||||
.eos_token_id
|
||||
.clone()
|
||||
.map(|value| value.into_set())
|
||||
.unwrap_or_default();
|
||||
if let Some(primary_eos_token_id) = primary_eos_token_id {
|
||||
extra_eos_token_ids.remove(&primary_eos_token_id);
|
||||
}
|
||||
let (primary_eos_token_id, extra_eos_token_ids) = resolve_eos_token_ids(
|
||||
&tokenizer_config,
|
||||
&model_config,
|
||||
&generation_config,
|
||||
tokenizer.as_ref(),
|
||||
);
|
||||
|
||||
info!(
|
||||
model_id,
|
||||
@@ -98,6 +92,42 @@ impl HfTextBackend {
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve EOS hints from tokenizer, model, and generation configs.
|
||||
///
|
||||
/// Resolution rules:
|
||||
/// 1. Use the tokenizer-side EOS token as the primary EOS when it resolves to a
|
||||
/// token id.
|
||||
/// 2. Fall back to the first model-config EOS id when the tokenizer config does
|
||||
/// not provide a primary EOS.
|
||||
/// 3. Keep any remaining model/generation EOS ids as extra stop-token ids so
|
||||
/// they still participate in stopping and min-token handling.
|
||||
fn resolve_eos_token_ids(
|
||||
tokenizer_config: &HfTokenizerConfig,
|
||||
model_config: &ModelConfig,
|
||||
generation_config: &GenerationConfig,
|
||||
tokenizer: &dyn Tokenizer,
|
||||
) -> (Option<u32>, BTreeSet<u32>) {
|
||||
let model_config_eos_token_ids = model_config.eos_token_ids();
|
||||
let primary_eos_token_id = tokenizer_config
|
||||
.special_tokens
|
||||
.eos_token
|
||||
.as_ref()
|
||||
.and_then(|token| tokenizer.token_to_id(token.as_str()))
|
||||
.or_else(|| model_config_eos_token_ids.first().copied());
|
||||
|
||||
let mut extra_eos_token_ids = generation_config
|
||||
.eos_token_id
|
||||
.clone()
|
||||
.map(|value| value.into_set())
|
||||
.unwrap_or_default();
|
||||
extra_eos_token_ids.extend(model_config_eos_token_ids.iter().copied());
|
||||
if let Some(primary_eos_token_id) = primary_eos_token_id {
|
||||
extra_eos_token_ids.remove(&primary_eos_token_id);
|
||||
}
|
||||
|
||||
(primary_eos_token_id, extra_eos_token_ids)
|
||||
}
|
||||
|
||||
impl TextBackend for HfTextBackend {
|
||||
fn tokenizer(&self) -> DynTokenizer {
|
||||
self.tokenizer.clone()
|
||||
@@ -128,3 +158,77 @@ impl TextBackend for HfTextBackend {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::{GenerationConfig, HfTokenizerConfig, ModelConfig, resolve_eos_token_ids};
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
_text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
_token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(String::new())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
(token == "</s>").then_some(2)
|
||||
}
|
||||
|
||||
fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
(id == 2).then(|| "</s>".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eos_resolution_uses_model_config_when_tokenizer_has_no_eos() {
|
||||
let tokenizer_config: HfTokenizerConfig = serde_json::from_str("{}").unwrap();
|
||||
let model_config: ModelConfig =
|
||||
serde_json::from_str(r#"{"eos_token_id":[200006,200010]}"#).unwrap();
|
||||
let generation_config: GenerationConfig = serde_json::from_str("{}").unwrap();
|
||||
|
||||
let (primary, extra) = resolve_eos_token_ids(
|
||||
&tokenizer_config,
|
||||
&model_config,
|
||||
&generation_config,
|
||||
&FakeTokenizer,
|
||||
);
|
||||
|
||||
assert_eq!(primary, Some(200006));
|
||||
assert_eq!(extra, BTreeSet::from([200010]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eos_resolution_keeps_tokenizer_eos_primary() {
|
||||
let tokenizer_config: HfTokenizerConfig =
|
||||
serde_json::from_str(r#"{"eos_token":"</s>"}"#).unwrap();
|
||||
let model_config: ModelConfig =
|
||||
serde_json::from_str(r#"{"eos_token_id":[2,200006]}"#).unwrap();
|
||||
let generation_config: GenerationConfig =
|
||||
serde_json::from_str(r#"{"eos_token_id":[2,200010]}"#).unwrap();
|
||||
|
||||
let (primary, extra) = resolve_eos_token_ids(
|
||||
&tokenizer_config,
|
||||
&model_config,
|
||||
&generation_config,
|
||||
&FakeTokenizer,
|
||||
);
|
||||
|
||||
assert_eq!(primary, Some(2));
|
||||
assert_eq!(extra, BTreeSet::from([200006, 200010]));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user