iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Rust Frontend] Support --generation-config vllm (#53044)

Browse Source 
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-08-20 17:31:25 -07:00
committed by GitHub
parent 0a5a55136f
commit f32b17b6d6
14 changed files with 244 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/chat/src/backend/hf.rs
+6 -2
View File
@@ -129,8 +129,11 @@ pub(super) async fn load_model_backends(
options: LoadModelBackendsOptions,
) -> Result<LoadedModelBackends> {
let files = ResolvedModelFiles::new(model_id).await?;
let text_backend =
HfTextBackend::from_resolved_model_files(files.clone(), model_id.to_string())?;
let text_backend = HfTextBackend::from_resolved_model_files(
files.clone(),
model_id.to_string(),
options.generation_config,
)?;
let tokenizer = text_backend.tokenizer();
let text_backend: DynTextBackend = Arc::new(text_backend);
@@ -227,6 +230,7 @@ mod tests {
resolved_files(config_json, tokenizer_config_json),
"test-model".to_string(),
LoadModelBackendsOptions {
generation_config: Default::default(),
renderer,
language_model_only: false,
chat_template_content_format: Default::default(),
rust/src/chat/src/backend/mod.rs
+3 -1
View File
@@ -5,7 +5,7 @@ use std::collections::HashMap;
use std::sync::Arc;
use serde_json::Value;
use vllm_text::{DynTextBackend, TextBackend};
use vllm_text::{DynTextBackend, GenerationConfigMode, TextBackend};
use crate::error::Result;
use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo};
@@ -61,6 +61,8 @@ pub type DynChatTextBackend = Arc<dyn ChatTextBackend>;
/// Frontend-side chat backend loading options.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct LoadModelBackendsOptions {
/// Which generation-config sampling defaults to inherit.
pub generation_config: GenerationConfigMode,
/// Which chat renderer implementation to use.
pub renderer: RendererSelection,
/// Disable frontend-side multimodal preprocessing and render the model as
rust/src/chat/src/lib.rs
+1
View File
@@ -44,6 +44,7 @@ pub use request::{
pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};
pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
pub use vllm_llm::FinishReason;
pub use vllm_text::GenerationConfigMode;
mod backend;
mod error;
rust/src/cmd/src/cli.rs
+9 -1
View File
@@ -22,8 +22,8 @@ use serde_json::Value;
use serde_with::{DefaultOnNull, OneOrMany, serde_as};
use thiserror_ext::AsReport as _;
use uuid::Uuid;
use vllm_chat::ReasoningParserFactory;
use vllm_chat::multimodal::MmLimitPerPrompt;
use vllm_chat::{GenerationConfigMode, ReasoningParserFactory};
use vllm_engine_core_client::TransportMode;
use vllm_managed_engine::ManagedEngineConfig;
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
@@ -186,6 +186,12 @@ pub struct SharedRuntimeArgs {
/// public model ID.
pub model: String,
/// The source of generation-config sampling defaults. `"auto"` loads the
/// model's defaults, while `"vllm"` uses vLLM's neutral defaults.
#[arg(long, default_value_t)]
#[serde(default)]
pub generation_config: GenerationConfigMode,
/// Maximum time to wait for the expected engines to register on the
/// frontend transport.
#[arg(
@@ -482,6 +488,7 @@ impl SharedRuntimeArgs {
None => CoordinatorMode::None,
},
model: self.model,
generation_config: self.generation_config,
served_model_name: self.served_model_name,
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
tool_call_parser: self.tool_call_parser,
@@ -535,6 +542,7 @@ impl SharedRuntimeArgs {
},
coordinator_mode: CoordinatorMode::MaybeInProc,
model: self.model,
generation_config: self.generation_config,
served_model_name: self.served_model_name,
listener_mode,
tool_call_parser: self.tool_call_parser,
rust/src/cmd/src/cli/tests.rs
+30 -2
View File
@@ -3,7 +3,9 @@
use expect_test::expect;
use vllm_engine_core_client::TransportMode;
use vllm_server::{Config, HttpListenerMode, ParserSelection, RendererSelection};
use vllm_server::{
Config, GenerationConfigMode, HttpListenerMode, ParserSelection, RendererSelection,
};
use super::{BenchCommand, Cli, Command};
@@ -88,6 +90,7 @@ fn serve_args_forward_python_flags_with_separator() {
uds: None,
runtime: SharedRuntimeArgs {
model: "Qwen/Qwen3-0.6B",
generation_config: Auto,
engine_ready_timeout_secs: 600,
tool_call_parser: Auto,
reasoning_parser: Auto,
@@ -732,6 +735,24 @@ fn serve_args_reject_unsupported_flag_arg() {
"#]].assert_eq(&error.to_string());
}
#[test]
fn serve_args_reject_custom_generation_config_source() {
let error = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--generation-config",
"/tmp/custom-config",
])
.unwrap_err();
expect![[r#"
error: invalid value '/tmp/custom-config' for '--generation-config <GENERATION_CONFIG>': generation config source `/tmp/custom-config` is not implemented yet (expected one of: auto, vllm)
For more information, try '--help'.
"#]].assert_eq(&error.to_string());
}
#[test]
fn serve_args_reject_unsupported_no_flag_alias() {
let error = Cli::try_parse_from([
@@ -788,6 +809,7 @@ fn frontend_args_accept_json() {
data_parallel_size: None,
runtime: SharedRuntimeArgs {
model: "Qwen/Qwen3-0.6B",
generation_config: Auto,
engine_ready_timeout_secs: 600,
tool_call_parser: None,
reasoning_parser: None,
@@ -858,6 +880,7 @@ fn frontend_args_json_applies_defaults() {
panic!("expected frontend args");
};
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
assert_eq!(args.runtime.generation_config, GenerationConfigMode::Auto);
assert_eq!(args.runtime.engine_ready_timeout_secs, 600);
assert_eq!(args.runtime.tool_call_parser, ParserSelection::None);
assert_eq!(args.runtime.reasoning_parser, ParserSelection::None);
@@ -901,7 +924,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
"--output-address",
"ipc:///tmp/output.sock",
"--args-json",
r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_logprobs":-1,"shutdown_timeout":3}"#,
r#"{"model_tag":"Qwen/Qwen3-0.6B","generation_config":"vllm","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_logprobs":-1,"shutdown_timeout":3}"#,
])
.unwrap();
@@ -909,6 +932,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
panic!("expected frontend args");
};
assert_eq!(args.runtime.engine_ready_timeout_secs, 42);
assert_eq!(args.runtime.generation_config, GenerationConfigMode::Vllm);
assert_eq!(
args.runtime.tool_call_parser,
ParserSelection::Explicit("hermes".to_string())
@@ -1379,6 +1403,7 @@ fn serve_args_accept_handshake_aliases() {
uds: None,
runtime: SharedRuntimeArgs {
model: "Qwen/Qwen3-0.6B",
generation_config: Auto,
engine_ready_timeout_secs: 600,
tool_call_parser: Auto,
reasoning_parser: Auto,
@@ -1523,6 +1548,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
},
coordinator_mode: MaybeInProc,
model: "Qwen/Qwen3-0.6B",
generation_config: Auto,
served_model_name: [],
listener_mode: BindTcp {
host: "127.0.0.1",
@@ -1608,6 +1634,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
},
coordinator_mode: MaybeInProc,
model: "Qwen/Qwen3-0.6B",
generation_config: Auto,
served_model_name: [],
listener_mode: BindTcp {
host: "127.0.0.1",
@@ -1715,6 +1742,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
address: "tcp://127.0.0.1:7000",
},
model: "Qwen/Qwen3-0.6B",
generation_config: Auto,
served_model_name: [],
listener_mode: InheritedFd {
fd: 3,
rust/src/cmd/src/cli/unsupported.rs
+5 -8
View File
@@ -240,15 +240,12 @@ pub struct EngineUnsupportedArgs {
#[arg(long)]
pub hf_overrides: Option<Unsupported>,
/// The folder path to the generation config. Defaults to `"auto"`, the
/// generation config will be loaded from model path. If set to `"vllm"`, no
/// generation config is loaded, vLLM defaults will be used. If set to a
/// folder path, the generation config will be loaded from the specified
/// folder path. If `max_new_tokens` is specified in generation config,
/// then it sets a server-wide limit on the number of output tokens for
/// all requests.
/// Overrides or sets generation config. e.g. `{"temperature": 0.5}`. If
/// used with `--generation-config auto`, the override parameters will be
/// merged with the default config from the model. If used with
/// `--generation-config vllm`, only the override parameters are used.
#[arg(long)]
pub generation_config: Option<Unsupported>,
pub override_generation_config: Option<Unsupported>,
/// IOProcessor plugin name to load at model startup
#[arg(long)]
rust/src/server/examples/external_engine_openai_qwen.rs
+1
View File
@@ -60,6 +60,7 @@ async fn main() -> Result<()> {
},
coordinator_mode: CoordinatorMode::MaybeInProc,
model: args.model,
generation_config: Default::default(),
served_model_name: vec![],
listener_mode: HttpListenerMode::BindTcp {
host: "127.0.0.1".to_string(),
rust/src/server/src/config.rs
+5 -1
View File
@@ -11,7 +11,9 @@ use educe::Educe;
use serde::Serialize;
use serde_json::Value;
use vllm_chat::multimodal::MmLimitPerPrompt;
use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
use vllm_chat::{
ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,
};
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
/// Default keep-alive idle timeout (seconds); also the head-read bound
@@ -165,6 +167,8 @@ pub struct Config {
pub coordinator_mode: CoordinatorMode,
/// Backend model identifier used for engine-core loading.
pub model: String,
/// Which generation-config sampling defaults to inherit.
pub generation_config: GenerationConfigMode,
/// Model name(s) exposed to clients via the OpenAI API. When non-empty,
/// the first entry is used as the primary ID in responses and all entries
/// are accepted in requests. When empty, falls back to `model`.
rust/src/server/src/lib.rs
+4 -1
View File
@@ -45,7 +45,9 @@ use tonic_health::server::health_reporter;
use tower::ServiceExt as _;
use tracing::{info, trace, warn};
use vllm_chat::{ChatLlm, LoadModelBackendsOptions, load_model_backends};
pub use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
pub use vllm_chat::{
ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,
};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
use vllm_llm::Llm;
use vllm_text::TextLlm;
@@ -96,6 +98,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
let loaded = load_model_backends(
&config.model,
LoadModelBackendsOptions {
generation_config: config.generation_config,
renderer: config.renderer,
language_model_only: config.language_model_only,
chat_template: config.chat_template.clone(),
rust/src/server/src/render.rs
+1
View File
@@ -56,6 +56,7 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
let loaded = load_model_backends(
&config.model,
LoadModelBackendsOptions {
generation_config: Default::default(),
renderer: config.renderer,
language_model_only: true,
chat_template: config.chat_template.clone(),
rust/src/text/src/backend/hf/mod.rs
+122 -19
View File
@@ -18,7 +18,7 @@ pub use self::config::{
load_tokenizer_config,
};
pub use self::model_files::{ResolvedModelFiles, TokenizerSource};
use crate::backend::{SamplingHints, TextBackend};
use crate::backend::{GenerationConfigMode, SamplingHints, TextBackend};
use crate::error::Result;
fn load_tokenizer(tokenizer: &TokenizerSource) -> Result<DynTokenizer> {
@@ -41,6 +41,7 @@ pub struct HfTextBackend {
/// Generation-config for sampling defaults that may be inherited when the
/// user does not explicitly override them.
generation_config: GenerationConfig,
generation_config_mode: GenerationConfigMode,
/// Model vocabulary size from the selected text config.
model_vocab_size: usize,
/// Model config (`config.json`).
@@ -48,14 +49,12 @@ pub struct HfTextBackend {
}
impl HfTextBackend {
/// Load the text backend with the given model id.
pub async fn from_model(model_id: &str) -> Result<Self> {
let files = ResolvedModelFiles::new(model_id).await?;
Self::from_resolved_model_files(files, model_id.to_string())
}
/// Load the text backend from resolved Hugging Face model files.
pub fn from_resolved_model_files(files: ResolvedModelFiles, model_id: String) -> Result<Self> {
pub fn from_resolved_model_files(
files: ResolvedModelFiles,
model_id: String,
generation_config_mode: GenerationConfigMode,
) -> Result<Self> {
let tokenizer_config = load_tokenizer_config(files.tokenizer_config_path.as_deref())?;
let tokenizer = load_tokenizer(&files.tokenizer)?;
let model_config = load_model_config(files.config_path.as_deref())?;
@@ -80,6 +79,7 @@ impl HfTextBackend {
primary_eos_token_id,
extra_eos_token_ids,
generation_config,
generation_config_mode,
model_vocab_size,
model_config,
})
@@ -146,16 +146,38 @@ impl TextBackend for HfTextBackend {
}
fn sampling_hints(&self) -> Result<SamplingHints> {
Ok(SamplingHints {
primary_eos_token_id: self.primary_eos_token_id,
extra_eos_token_ids: self.extra_eos_token_ids.clone(),
default_temperature: self.generation_config.temperature,
default_top_p: self.generation_config.top_p,
default_top_k: self.generation_config.top_k,
default_min_p: self.generation_config.min_p,
default_repetition_penalty: self.generation_config.repetition_penalty,
default_max_tokens: self.generation_config.max_new_tokens,
})
Ok(build_sampling_hints(
self.generation_config_mode,
&self.generation_config,
self.primary_eos_token_id,
self.extra_eos_token_ids.clone(),
))
}
}
fn build_sampling_hints(
mode: GenerationConfigMode,
generation_config: &GenerationConfig,
primary_eos_token_id: Option<u32>,
extra_eos_token_ids: BTreeSet<u32>,
) -> SamplingHints {
let sampling_config = match mode {
// Auto inherits sampling defaults from the model's generation config.
GenerationConfigMode::Auto => Some(generation_config),
// Vllm skips model-provided sampling defaults. `lower_sampling_params`
// applies vLLM fallback values; separately resolved EOS tokens remain.
GenerationConfigMode::Vllm => None,
};
SamplingHints {
primary_eos_token_id,
extra_eos_token_ids,
default_temperature: sampling_config.and_then(|config| config.temperature),
default_top_p: sampling_config.and_then(|config| config.top_p),
default_top_k: sampling_config.and_then(|config| config.top_k),
default_min_p: sampling_config.and_then(|config| config.min_p),
default_repetition_penalty: sampling_config.and_then(|config| config.repetition_penalty),
default_max_tokens: sampling_config.and_then(|config| config.max_new_tokens),
}
}
@@ -163,7 +185,10 @@ impl TextBackend for HfTextBackend {
mod tests {
use std::collections::BTreeSet;
use super::{GenerationConfig, HfTokenizerConfig, ModelConfig, resolve_eos_token_ids};
use super::{
GenerationConfig, GenerationConfigMode, HfTokenizerConfig, ModelConfig,
build_sampling_hints, resolve_eos_token_ids,
};
use vllm_tokenizer::Tokenizer;
struct FakeTokenizer;
@@ -235,4 +260,82 @@ mod tests {
assert_eq!(primary, Some(2));
assert_eq!(extra, BTreeSet::from([200006, 200010]));
}
#[test]
fn vllm_generation_config_keeps_eos_and_uses_neutral_sampling_defaults() {
let generation_config: GenerationConfig = serde_json::from_str(
r#"{
"eos_token_id": [2, 3],
"temperature": 0.6,
"top_p": 0.95,
"top_k": 20,
"min_p": 0.1,
"repetition_penalty": 1.1,
"max_new_tokens": 4096
}"#,
)
.unwrap();
let extra_eos_token_ids = BTreeSet::from([3]);
let hints = (
build_sampling_hints(
GenerationConfigMode::Auto,
&generation_config,
Some(2),
extra_eos_token_ids.clone(),
),
build_sampling_hints(
GenerationConfigMode::Vllm,
&generation_config,
Some(2),
extra_eos_token_ids,
),
);
expect_test::expect![[r#"
(
SamplingHints {
primary_eos_token_id: Some(
2,
),
extra_eos_token_ids: {
3,
},
default_temperature: Some(
0.6,
),
default_top_p: Some(
0.95,
),
default_top_k: Some(
20,
),
default_min_p: Some(
0.1,
),
default_repetition_penalty: Some(
1.1,
),
default_max_tokens: Some(
4096,
),
},
SamplingHints {
primary_eos_token_id: Some(
2,
),
extra_eos_token_ids: {
3,
},
default_temperature: None,
default_top_p: None,
default_top_k: None,
default_min_p: None,
default_repetition_penalty: None,
default_max_tokens: None,
},
)
"#]]
.assert_debug_eq(&hints);
}
}
rust/src/text/src/backend/mod.rs
+44
View File
@@ -3,12 +3,56 @@
pub mod hf;
use std::fmt;
use std::str::FromStr;
use std::sync::Arc;
use serde_with::{DeserializeFromStr, SerializeDisplay};
use vllm_tokenizer::DynTokenizer;
use crate::error::Result;
/// Select which sampling defaults to inherit from generation config.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]
pub enum GenerationConfigMode {
/// Inherit sampling defaults from the model's generation config.
#[default]
Auto,
/// Use vLLM's neutral sampling defaults.
Vllm,
}
impl GenerationConfigMode {
pub const AUTO_LITERAL: &str = "auto";
pub const VLLM_LITERAL: &str = "vllm";
}
impl FromStr for GenerationConfigMode {
type Err = String;
fn from_str(value: &str) -> std::result::Result<Self, Self::Err> {
if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {
Ok(Self::Auto)
} else if value.eq_ignore_ascii_case(Self::VLLM_LITERAL) {
Ok(Self::Vllm)
} else {
Err(format!(
"generation config source `{value}` is not implemented yet \
(expected one of: auto, vllm)"
))
}
}
}
impl fmt::Display for GenerationConfigMode {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Auto => f.write_str(Self::AUTO_LITERAL),
Self::Vllm => f.write_str(Self::VLLM_LITERAL),
}
}
}
/// Tokenizer/model-derived defaults used to enrich text-generation requests
/// before they are lowered into engine-core.
#[derive(Debug, Clone, Default, PartialEq)]
rust/src/text/src/lib.rs
+3 -1
View File
@@ -9,7 +9,9 @@
use std::mem::take;
pub use backend::{DynTextBackend, SamplingHints, SamplingLimits, TextBackend};
pub use backend::{
DynTextBackend, GenerationConfigMode, SamplingHints, SamplingLimits, TextBackend,
};
pub use error::{Error, LogprobsError, Result, SamplingParamsError, TokenIdsError};
use futures::Stream;
pub use lower::{
rust/src/text/src/lower.rs
+10 -5
View File
@@ -322,8 +322,8 @@ mod tests {
use vllm_tokenizer::test_utils::TestTokenizer;
use super::*;
use crate::backend::hf::HfTextBackend;
use crate::backend::{SamplingHints, TextBackend as _};
use crate::backend::hf::{HfTextBackend, ResolvedModelFiles};
use crate::backend::{GenerationConfigMode, SamplingHints, TextBackend as _};
use crate::error::{LogprobsError, SamplingParamsError, TokenIdsError};
use crate::request::{Prompt, TextRequest};
@@ -785,9 +785,14 @@ mod tests {
#[tokio::test]
#[file_serial(hf_qwen3)]
async fn lower_text_request_uses_real_qwen_generation_defaults() {
let backend = HfTextBackend::from_model("Qwen/Qwen3-0.6B")
.await
.expect("load qwen tokenizer and generation config");
let model_id = "Qwen/Qwen3-0.6B";
let files = ResolvedModelFiles::new(model_id).await.expect("resolve qwen model files");
let backend = HfTextBackend::from_resolved_model_files(
files,
model_id.to_string(),
GenerationConfigMode::Auto,
)
.expect("load qwen tokenizer and generation config");
let hints = backend.sampling_hints().expect("collect sampling hints");
expect_test::expect![[r#"