[Rust Frontend] Support --generation-config vllm (#53044)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-08-20 17:31:25 -07:00
committed by GitHub
parent 0a5a55136f
commit f32b17b6d6
14 changed files with 244 additions and 41 deletions
+122 -19
View File
@@ -18,7 +18,7 @@ pub use self::config::{
load_tokenizer_config,
};
pub use self::model_files::{ResolvedModelFiles, TokenizerSource};
use crate::backend::{SamplingHints, TextBackend};
use crate::backend::{GenerationConfigMode, SamplingHints, TextBackend};
use crate::error::Result;
fn load_tokenizer(tokenizer: &TokenizerSource) -> Result<DynTokenizer> {
@@ -41,6 +41,7 @@ pub struct HfTextBackend {
/// Generation-config for sampling defaults that may be inherited when the
/// user does not explicitly override them.
generation_config: GenerationConfig,
generation_config_mode: GenerationConfigMode,
/// Model vocabulary size from the selected text config.
model_vocab_size: usize,
/// Model config (`config.json`).
@@ -48,14 +49,12 @@ pub struct HfTextBackend {
}
impl HfTextBackend {
/// Load the text backend with the given model id.
pub async fn from_model(model_id: &str) -> Result<Self> {
let files = ResolvedModelFiles::new(model_id).await?;
Self::from_resolved_model_files(files, model_id.to_string())
}
/// Load the text backend from resolved Hugging Face model files.
pub fn from_resolved_model_files(files: ResolvedModelFiles, model_id: String) -> Result<Self> {
pub fn from_resolved_model_files(
files: ResolvedModelFiles,
model_id: String,
generation_config_mode: GenerationConfigMode,
) -> Result<Self> {
let tokenizer_config = load_tokenizer_config(files.tokenizer_config_path.as_deref())?;
let tokenizer = load_tokenizer(&files.tokenizer)?;
let model_config = load_model_config(files.config_path.as_deref())?;
@@ -80,6 +79,7 @@ impl HfTextBackend {
primary_eos_token_id,
extra_eos_token_ids,
generation_config,
generation_config_mode,
model_vocab_size,
model_config,
})
@@ -146,16 +146,38 @@ impl TextBackend for HfTextBackend {
}
fn sampling_hints(&self) -> Result<SamplingHints> {
Ok(SamplingHints {
primary_eos_token_id: self.primary_eos_token_id,
extra_eos_token_ids: self.extra_eos_token_ids.clone(),
default_temperature: self.generation_config.temperature,
default_top_p: self.generation_config.top_p,
default_top_k: self.generation_config.top_k,
default_min_p: self.generation_config.min_p,
default_repetition_penalty: self.generation_config.repetition_penalty,
default_max_tokens: self.generation_config.max_new_tokens,
})
Ok(build_sampling_hints(
self.generation_config_mode,
&self.generation_config,
self.primary_eos_token_id,
self.extra_eos_token_ids.clone(),
))
}
}
fn build_sampling_hints(
mode: GenerationConfigMode,
generation_config: &GenerationConfig,
primary_eos_token_id: Option<u32>,
extra_eos_token_ids: BTreeSet<u32>,
) -> SamplingHints {
let sampling_config = match mode {
// Auto inherits sampling defaults from the model's generation config.
GenerationConfigMode::Auto => Some(generation_config),
// Vllm skips model-provided sampling defaults. `lower_sampling_params`
// applies vLLM fallback values; separately resolved EOS tokens remain.
GenerationConfigMode::Vllm => None,
};
SamplingHints {
primary_eos_token_id,
extra_eos_token_ids,
default_temperature: sampling_config.and_then(|config| config.temperature),
default_top_p: sampling_config.and_then(|config| config.top_p),
default_top_k: sampling_config.and_then(|config| config.top_k),
default_min_p: sampling_config.and_then(|config| config.min_p),
default_repetition_penalty: sampling_config.and_then(|config| config.repetition_penalty),
default_max_tokens: sampling_config.and_then(|config| config.max_new_tokens),
}
}
@@ -163,7 +185,10 @@ impl TextBackend for HfTextBackend {
mod tests {
use std::collections::BTreeSet;
use super::{GenerationConfig, HfTokenizerConfig, ModelConfig, resolve_eos_token_ids};
use super::{
GenerationConfig, GenerationConfigMode, HfTokenizerConfig, ModelConfig,
build_sampling_hints, resolve_eos_token_ids,
};
use vllm_tokenizer::Tokenizer;
struct FakeTokenizer;
@@ -235,4 +260,82 @@ mod tests {
assert_eq!(primary, Some(2));
assert_eq!(extra, BTreeSet::from([200006, 200010]));
}
#[test]
fn vllm_generation_config_keeps_eos_and_uses_neutral_sampling_defaults() {
let generation_config: GenerationConfig = serde_json::from_str(
r#"{
"eos_token_id": [2, 3],
"temperature": 0.6,
"top_p": 0.95,
"top_k": 20,
"min_p": 0.1,
"repetition_penalty": 1.1,
"max_new_tokens": 4096
}"#,
)
.unwrap();
let extra_eos_token_ids = BTreeSet::from([3]);
let hints = (
build_sampling_hints(
GenerationConfigMode::Auto,
&generation_config,
Some(2),
extra_eos_token_ids.clone(),
),
build_sampling_hints(
GenerationConfigMode::Vllm,
&generation_config,
Some(2),
extra_eos_token_ids,
),
);
expect_test::expect![[r#"
(
SamplingHints {
primary_eos_token_id: Some(
2,
),
extra_eos_token_ids: {
3,
},
default_temperature: Some(
0.6,
),
default_top_p: Some(
0.95,
),
default_top_k: Some(
20,
),
default_min_p: Some(
0.1,
),
default_repetition_penalty: Some(
1.1,
),
default_max_tokens: Some(
4096,
),
},
SamplingHints {
primary_eos_token_id: Some(
2,
),
extra_eos_token_ids: {
3,
},
default_temperature: None,
default_top_p: None,
default_top_k: None,
default_min_p: None,
default_repetition_penalty: None,
default_max_tokens: None,
},
)
"#]]
.assert_debug_eq(&hints);
}
}
+44
View File
@@ -3,12 +3,56 @@
pub mod hf;
use std::fmt;
use std::str::FromStr;
use std::sync::Arc;
use serde_with::{DeserializeFromStr, SerializeDisplay};
use vllm_tokenizer::DynTokenizer;
use crate::error::Result;
/// Select which sampling defaults to inherit from generation config.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]
pub enum GenerationConfigMode {
/// Inherit sampling defaults from the model's generation config.
#[default]
Auto,
/// Use vLLM's neutral sampling defaults.
Vllm,
}
impl GenerationConfigMode {
pub const AUTO_LITERAL: &str = "auto";
pub const VLLM_LITERAL: &str = "vllm";
}
impl FromStr for GenerationConfigMode {
type Err = String;
fn from_str(value: &str) -> std::result::Result<Self, Self::Err> {
if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {
Ok(Self::Auto)
} else if value.eq_ignore_ascii_case(Self::VLLM_LITERAL) {
Ok(Self::Vllm)
} else {
Err(format!(
"generation config source `{value}` is not implemented yet \
(expected one of: auto, vllm)"
))
}
}
}
impl fmt::Display for GenerationConfigMode {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Auto => f.write_str(Self::AUTO_LITERAL),
Self::Vllm => f.write_str(Self::VLLM_LITERAL),
}
}
}
/// Tokenizer/model-derived defaults used to enrich text-generation requests
/// before they are lowered into engine-core.
#[derive(Debug, Clone, Default, PartialEq)]
+3 -1
View File
@@ -9,7 +9,9 @@
use std::mem::take;
pub use backend::{DynTextBackend, SamplingHints, SamplingLimits, TextBackend};
pub use backend::{
DynTextBackend, GenerationConfigMode, SamplingHints, SamplingLimits, TextBackend,
};
pub use error::{Error, LogprobsError, Result, SamplingParamsError, TokenIdsError};
use futures::Stream;
pub use lower::{
+10 -5
View File
@@ -322,8 +322,8 @@ mod tests {
use vllm_tokenizer::test_utils::TestTokenizer;
use super::*;
use crate::backend::hf::HfTextBackend;
use crate::backend::{SamplingHints, TextBackend as _};
use crate::backend::hf::{HfTextBackend, ResolvedModelFiles};
use crate::backend::{GenerationConfigMode, SamplingHints, TextBackend as _};
use crate::error::{LogprobsError, SamplingParamsError, TokenIdsError};
use crate::request::{Prompt, TextRequest};
@@ -785,9 +785,14 @@ mod tests {
#[tokio::test]
#[file_serial(hf_qwen3)]
async fn lower_text_request_uses_real_qwen_generation_defaults() {
let backend = HfTextBackend::from_model("Qwen/Qwen3-0.6B")
.await
.expect("load qwen tokenizer and generation config");
let model_id = "Qwen/Qwen3-0.6B";
let files = ResolvedModelFiles::new(model_id).await.expect("resolve qwen model files");
let backend = HfTextBackend::from_resolved_model_files(
files,
model_id.to_string(),
GenerationConfigMode::Auto,
)
.expect("load qwen tokenizer and generation config");
let hints = backend.sampling_hints().expect("collect sampling hints");
expect_test::expect![[r#"