[Rust Frontend] Support HF config overrides --hf-overrides (#56931)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -33,6 +33,7 @@ vllm-chat.workspace = true
|
||||
vllm-engine-core-client.workspace = true
|
||||
vllm-managed-engine.workspace = true
|
||||
vllm-server.workspace = true
|
||||
vllm-text.workspace = true
|
||||
vllm-tracing.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -33,6 +33,7 @@ use vllm_server::{
|
||||
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, LoraModulePath, ParserSelection, RenderConfig,
|
||||
RendererSelection,
|
||||
};
|
||||
use vllm_text::backend::hf::HfOverrides;
|
||||
|
||||
use crate::cli::ssl::SslArgs;
|
||||
use crate::cli::unsupported::UnsupportedArgs;
|
||||
@@ -113,6 +114,9 @@ pub struct RenderArgs {
|
||||
/// Model revision on the Hugging Face Hub (branch, tag, or commit SHA).
|
||||
#[arg(long)]
|
||||
revision: Option<String>,
|
||||
/// JSON Merge Patch (RFC 7396) for config.json; null removes a field.
|
||||
#[arg(long, value_parser = parse_json::<HfOverrides>, default_value = "{}", value_name = "JSON")]
|
||||
hf_overrides: HfOverrides,
|
||||
/// HTTP bind host.
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
@@ -161,6 +165,7 @@ impl RenderArgs {
|
||||
RenderConfig {
|
||||
model: self.model,
|
||||
revision: self.revision,
|
||||
hf_overrides: self.hf_overrides,
|
||||
served_model_name: self.served_model_name,
|
||||
host: self.host,
|
||||
port: self.port,
|
||||
@@ -204,6 +209,12 @@ pub struct SharedRuntimeArgs {
|
||||
#[serde(default)]
|
||||
pub revision: Option<String>,
|
||||
|
||||
/// JSON Merge Patch (RFC 7396) for config.json; null removes a field.
|
||||
/// Objects merge recursively and arrays/scalars replace existing values.
|
||||
#[arg(long, value_parser = parse_json::<HfOverrides>, default_value = "{}", value_name = "JSON")]
|
||||
#[serde(default)]
|
||||
pub hf_overrides: HfOverrides,
|
||||
|
||||
/// The source of generation-config sampling defaults. `"auto"` loads the
|
||||
/// model's defaults, while `"vllm"` uses vLLM's neutral defaults.
|
||||
#[arg(long, default_value_t)]
|
||||
@@ -500,6 +511,7 @@ impl SharedRuntimeArgs {
|
||||
},
|
||||
model: self.model,
|
||||
revision: self.revision,
|
||||
hf_overrides: self.hf_overrides,
|
||||
generation_config: self.generation_config,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
|
||||
@@ -556,6 +568,7 @@ impl SharedRuntimeArgs {
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: self.model,
|
||||
revision: self.revision,
|
||||
hf_overrides: self.hf_overrides,
|
||||
generation_config: self.generation_config,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode,
|
||||
@@ -761,6 +774,9 @@ impl ServeArgs {
|
||||
let reasoning_parser =
|
||||
effective_engine_reasoning_parser(&self.runtime.reasoning_parser, &self.runtime.model);
|
||||
let profiler_config = self.runtime.profiler_config_json();
|
||||
let hf_overrides = (!self.runtime.hf_overrides.is_empty()).then(|| {
|
||||
serde_json::to_string(&self.runtime.hf_overrides).expect("JSON object serializes")
|
||||
});
|
||||
|
||||
self.managed_engine.clone().into_config(
|
||||
self.runtime.model.clone(),
|
||||
@@ -773,6 +789,7 @@ impl ServeArgs {
|
||||
self.runtime.shutdown_timeout,
|
||||
handshake_port,
|
||||
self.runtime.limit_mm_per_prompt_json(),
|
||||
hf_overrides,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,6 +177,9 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
hf_overrides: HfOverrides(
|
||||
{},
|
||||
),
|
||||
generation_config: Auto,
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
@@ -1013,6 +1016,9 @@ fn frontend_args_accept_json() {
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
hf_overrides: HfOverrides(
|
||||
{},
|
||||
),
|
||||
generation_config: Auto,
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: None,
|
||||
@@ -1120,6 +1126,73 @@ fn frontend_args_json_ignores_engine_owned_max_model_len() {
|
||||
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hf_overrides_preserve_merge_patch_in_cli_and_python_bootstrap() {
|
||||
let patch = r#"{"text_config":{"rope_parameters":{"factor":4}},"sliding_window":null}"#;
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--hf-overrides",
|
||||
patch,
|
||||
])
|
||||
.unwrap();
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args")
|
||||
};
|
||||
let expected: serde_json::Value = serde_json::from_str(patch).unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(&args.runtime.hf_overrides).unwrap(),
|
||||
expected
|
||||
);
|
||||
let engine = args.to_managed_engine_config(1234);
|
||||
let index = engine.python_args.iter().position(|arg| arg == "--hf-overrides").unwrap();
|
||||
assert_eq!(
|
||||
serde_json::from_str::<serde_json::Value>(&engine.python_args[index + 1]).unwrap(),
|
||||
expected
|
||||
);
|
||||
assert!(!engine.python_args.iter().any(|arg| arg == "--hf-config-path"));
|
||||
let config = args.to_frontend_config("tcp://127.0.0.1:1234".to_string());
|
||||
assert_eq!(serde_json::to_value(config.hf_overrides).unwrap(), expected);
|
||||
|
||||
let payload = serde_json::json!({"model_tag":"Qwen/Qwen3-0.6B", "hf_overrides":expected});
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
&payload.to_string(),
|
||||
])
|
||||
.unwrap();
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args")
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_value(args.into_config().hf_overrides).unwrap(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hf_overrides_reject_non_object_inputs() {
|
||||
for patch in ["null", "[]", "1", r#""callable""#] {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--hf-overrides",
|
||||
patch,
|
||||
])
|
||||
.unwrap_err();
|
||||
assert_eq!(error.kind(), clap::error::ErrorKind::ValueValidation);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_accepts_supported_non_default_fields() {
|
||||
let cli = Cli::try_parse_from([
|
||||
@@ -1613,6 +1686,9 @@ fn serve_args_accept_handshake_aliases() {
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
hf_overrides: HfOverrides(
|
||||
{},
|
||||
),
|
||||
generation_config: Auto,
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
@@ -1763,6 +1839,9 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
hf_overrides: HfOverrides(
|
||||
{},
|
||||
),
|
||||
generation_config: Auto,
|
||||
served_model_name: [],
|
||||
listener_mode: BindTcp {
|
||||
@@ -1852,6 +1931,9 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
hf_overrides: HfOverrides(
|
||||
{},
|
||||
),
|
||||
generation_config: Auto,
|
||||
served_model_name: [],
|
||||
listener_mode: BindTcp {
|
||||
@@ -1963,6 +2045,9 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
},
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
hf_overrides: HfOverrides(
|
||||
{},
|
||||
),
|
||||
generation_config: Auto,
|
||||
served_model_name: [],
|
||||
listener_mode: InheritedFd {
|
||||
|
||||
@@ -235,11 +235,6 @@ pub struct EngineUnsupportedArgs {
|
||||
#[arg(long, default_missing_value = "true", num_args = 0..=1)]
|
||||
pub hf_token: Option<Unsupported>,
|
||||
|
||||
/// If a dictionary, contains arguments to be forwarded to the Hugging Face
|
||||
/// config. If a callable, it is called to update the HuggingFace config.
|
||||
#[arg(long)]
|
||||
pub hf_overrides: Option<Unsupported>,
|
||||
|
||||
/// Overrides or sets generation config. e.g. `{"temperature": 0.5}`. If
|
||||
/// used with `--generation-config auto`, the override parameters will be
|
||||
/// merged with the default config from the model. If used with
|
||||
|
||||
Reference in New Issue
Block a user