[Rust Frontend] Support HF config overrides --hf-overrides (#56931)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-16 02:52:55 +00:00
committed by GitHub
parent 0673de9800
commit bfd713bf8b
16 changed files with 298 additions and 19 deletions
@@ -61,6 +61,7 @@ async fn main() -> Result<()> {
coordinator_mode: CoordinatorMode::MaybeInProc,
model: args.model,
revision: None,
hf_overrides: Default::default(),
generation_config: Default::default(),
served_model_name: vec![],
listener_mode: HttpListenerMode::BindTcp {
+3
View File
@@ -16,6 +16,7 @@ use vllm_chat::{
ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,
};
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
use vllm_text::backend::hf::HfOverrides;
/// Default keep-alive idle timeout (seconds); also the head-read bound
/// when keep-alive is disabled (`0`).
@@ -214,6 +215,8 @@ pub struct Config {
pub model: String,
/// Model revision on the Hugging Face Hub (branch, tag, or commit SHA).
pub revision: Option<String>,
/// JSON Merge Patch applied to the model config before backend initialization.
pub hf_overrides: HfOverrides,
/// Which generation-config sampling defaults to inherit.
pub generation_config: GenerationConfigMode,
/// Model name(s) exposed to clients via the OpenAI API. When non-empty,
+1
View File
@@ -99,6 +99,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
&config.model,
LoadModelBackendsOptions {
revision: config.revision.clone(),
hf_overrides: config.hf_overrides.clone(),
generation_config: config.generation_config,
renderer: config.renderer,
language_model_only: config.language_model_only,
+4
View File
@@ -13,6 +13,7 @@ use vllm_chat::{
ParserSelection, RendererSelection, load_model_backends,
};
use vllm_text::TextRequestProcessor;
use vllm_text::backend::hf::HfOverrides;
use crate::{
HttpListenerMode, TlsConfig,
@@ -25,6 +26,7 @@ use crate::{
pub struct RenderConfig {
pub model: String,
pub revision: Option<String>,
pub hf_overrides: HfOverrides,
pub served_model_name: Vec<String>,
pub host: String,
pub port: u16,
@@ -69,6 +71,7 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
&config.model,
LoadModelBackendsOptions {
revision: config.revision.clone(),
hf_overrides: config.hf_overrides.clone(),
generation_config: Default::default(),
renderer: config.renderer,
language_model_only: true,
@@ -153,6 +156,7 @@ mod tests {
RenderConfig {
model: "test-model".to_string(),
revision: None,
hf_overrides: Default::default(),
served_model_name: Vec::new(),
host: "127.0.0.1".to_string(),
port: 8000,