[Rust Frontend] Support HF config overrides --hf-overrides (#56931)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Generated
+1
@@ -5813,6 +5813,7 @@ dependencies = [
|
||||
"vllm-engine-core-client",
|
||||
"vllm-managed-engine",
|
||||
"vllm-server",
|
||||
"vllm-text",
|
||||
"vllm-tracing",
|
||||
]
|
||||
|
||||
|
||||
@@ -129,7 +129,8 @@ pub(super) async fn load_model_backends(
|
||||
model_id: &str,
|
||||
options: LoadModelBackendsOptions,
|
||||
) -> Result<LoadedModelBackends> {
|
||||
let files = ResolvedModelFiles::new(model_id, options.revision.as_deref()).await?;
|
||||
let mut files = ResolvedModelFiles::new(model_id, options.revision.as_deref()).await?;
|
||||
files.apply_overrides(&options.hf_overrides)?;
|
||||
let text_backend = HfTextBackend::from_resolved_model_files(
|
||||
files.clone(),
|
||||
model_id.to_string(),
|
||||
@@ -166,13 +167,11 @@ fn resolve_multimodal_render_info(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use tempfile::tempdir;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use vllm_text::Prompt;
|
||||
use vllm_text::backend::hf::TokenizerSource;
|
||||
use vllm_text::tokenizer::DynTokenizer;
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
@@ -206,16 +205,12 @@ mod tests {
|
||||
write_json(&config_path, config_json);
|
||||
write_json(&tokenizer_config_path, tokenizer_config_json);
|
||||
|
||||
vllm_text::backend::hf::ResolvedModelFiles {
|
||||
tokenizer: TokenizerSource::HuggingFace(PathBuf::from("/tmp/unused-tokenizer.json")),
|
||||
tokenizer_config_path: Some(tokenizer_config_path),
|
||||
generation_config_path: None,
|
||||
preprocessor_config_path: None,
|
||||
video_preprocessor_config_path: None,
|
||||
processor_config_path: None,
|
||||
chat_template_path: None,
|
||||
config_path: Some(config_path),
|
||||
}
|
||||
write_json(&root.join("tokenizer.json"), "{}");
|
||||
futures::executor::block_on(vllm_text::backend::hf::ResolvedModelFiles::new(
|
||||
root.to_str().unwrap(),
|
||||
None,
|
||||
))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn test_tokenizer() -> DynTokenizer {
|
||||
@@ -232,6 +227,7 @@ mod tests {
|
||||
"test-model".to_string(),
|
||||
LoadModelBackendsOptions {
|
||||
revision: None,
|
||||
hf_overrides: Default::default(),
|
||||
generation_config: Default::default(),
|
||||
renderer,
|
||||
language_model_only: false,
|
||||
|
||||
@@ -5,6 +5,7 @@ use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::Value;
|
||||
use vllm_text::backend::hf::HfOverrides;
|
||||
use vllm_text::{DynTextBackend, GenerationConfigMode, TextBackend};
|
||||
|
||||
use crate::error::Result;
|
||||
@@ -63,6 +64,8 @@ pub type DynChatTextBackend = Arc<dyn ChatTextBackend>;
|
||||
pub struct LoadModelBackendsOptions {
|
||||
/// Model revision on the Hugging Face Hub (branch, tag, or commit SHA).
|
||||
pub revision: Option<String>,
|
||||
/// JSON Merge Patch applied to the model config before loading any backend.
|
||||
pub hf_overrides: HfOverrides,
|
||||
/// Which generation-config sampling defaults to inherit.
|
||||
pub generation_config: GenerationConfigMode,
|
||||
/// Which chat renderer implementation to use.
|
||||
|
||||
@@ -33,6 +33,7 @@ vllm-chat.workspace = true
|
||||
vllm-engine-core-client.workspace = true
|
||||
vllm-managed-engine.workspace = true
|
||||
vllm-server.workspace = true
|
||||
vllm-text.workspace = true
|
||||
vllm-tracing.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -33,6 +33,7 @@ use vllm_server::{
|
||||
DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, LoraModulePath, ParserSelection, RenderConfig,
|
||||
RendererSelection,
|
||||
};
|
||||
use vllm_text::backend::hf::HfOverrides;
|
||||
|
||||
use crate::cli::ssl::SslArgs;
|
||||
use crate::cli::unsupported::UnsupportedArgs;
|
||||
@@ -113,6 +114,9 @@ pub struct RenderArgs {
|
||||
/// Model revision on the Hugging Face Hub (branch, tag, or commit SHA).
|
||||
#[arg(long)]
|
||||
revision: Option<String>,
|
||||
/// JSON Merge Patch (RFC 7396) for config.json; null removes a field.
|
||||
#[arg(long, value_parser = parse_json::<HfOverrides>, default_value = "{}", value_name = "JSON")]
|
||||
hf_overrides: HfOverrides,
|
||||
/// HTTP bind host.
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
@@ -161,6 +165,7 @@ impl RenderArgs {
|
||||
RenderConfig {
|
||||
model: self.model,
|
||||
revision: self.revision,
|
||||
hf_overrides: self.hf_overrides,
|
||||
served_model_name: self.served_model_name,
|
||||
host: self.host,
|
||||
port: self.port,
|
||||
@@ -204,6 +209,12 @@ pub struct SharedRuntimeArgs {
|
||||
#[serde(default)]
|
||||
pub revision: Option<String>,
|
||||
|
||||
/// JSON Merge Patch (RFC 7396) for config.json; null removes a field.
|
||||
/// Objects merge recursively and arrays/scalars replace existing values.
|
||||
#[arg(long, value_parser = parse_json::<HfOverrides>, default_value = "{}", value_name = "JSON")]
|
||||
#[serde(default)]
|
||||
pub hf_overrides: HfOverrides,
|
||||
|
||||
/// The source of generation-config sampling defaults. `"auto"` loads the
|
||||
/// model's defaults, while `"vllm"` uses vLLM's neutral defaults.
|
||||
#[arg(long, default_value_t)]
|
||||
@@ -500,6 +511,7 @@ impl SharedRuntimeArgs {
|
||||
},
|
||||
model: self.model,
|
||||
revision: self.revision,
|
||||
hf_overrides: self.hf_overrides,
|
||||
generation_config: self.generation_config,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
|
||||
@@ -556,6 +568,7 @@ impl SharedRuntimeArgs {
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: self.model,
|
||||
revision: self.revision,
|
||||
hf_overrides: self.hf_overrides,
|
||||
generation_config: self.generation_config,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode,
|
||||
@@ -761,6 +774,9 @@ impl ServeArgs {
|
||||
let reasoning_parser =
|
||||
effective_engine_reasoning_parser(&self.runtime.reasoning_parser, &self.runtime.model);
|
||||
let profiler_config = self.runtime.profiler_config_json();
|
||||
let hf_overrides = (!self.runtime.hf_overrides.is_empty()).then(|| {
|
||||
serde_json::to_string(&self.runtime.hf_overrides).expect("JSON object serializes")
|
||||
});
|
||||
|
||||
self.managed_engine.clone().into_config(
|
||||
self.runtime.model.clone(),
|
||||
@@ -773,6 +789,7 @@ impl ServeArgs {
|
||||
self.runtime.shutdown_timeout,
|
||||
handshake_port,
|
||||
self.runtime.limit_mm_per_prompt_json(),
|
||||
hf_overrides,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,6 +177,9 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
hf_overrides: HfOverrides(
|
||||
{},
|
||||
),
|
||||
generation_config: Auto,
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
@@ -1013,6 +1016,9 @@ fn frontend_args_accept_json() {
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
hf_overrides: HfOverrides(
|
||||
{},
|
||||
),
|
||||
generation_config: Auto,
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: None,
|
||||
@@ -1120,6 +1126,73 @@ fn frontend_args_json_ignores_engine_owned_max_model_len() {
|
||||
assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hf_overrides_preserve_merge_patch_in_cli_and_python_bootstrap() {
|
||||
let patch = r#"{"text_config":{"rope_parameters":{"factor":4}},"sliding_window":null}"#;
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--hf-overrides",
|
||||
patch,
|
||||
])
|
||||
.unwrap();
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args")
|
||||
};
|
||||
let expected: serde_json::Value = serde_json::from_str(patch).unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(&args.runtime.hf_overrides).unwrap(),
|
||||
expected
|
||||
);
|
||||
let engine = args.to_managed_engine_config(1234);
|
||||
let index = engine.python_args.iter().position(|arg| arg == "--hf-overrides").unwrap();
|
||||
assert_eq!(
|
||||
serde_json::from_str::<serde_json::Value>(&engine.python_args[index + 1]).unwrap(),
|
||||
expected
|
||||
);
|
||||
assert!(!engine.python_args.iter().any(|arg| arg == "--hf-config-path"));
|
||||
let config = args.to_frontend_config("tcp://127.0.0.1:1234".to_string());
|
||||
assert_eq!(serde_json::to_value(config.hf_overrides).unwrap(), expected);
|
||||
|
||||
let payload = serde_json::json!({"model_tag":"Qwen/Qwen3-0.6B", "hf_overrides":expected});
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"frontend",
|
||||
"--listen-fd",
|
||||
"3",
|
||||
"--input-address",
|
||||
"ipc:///tmp/input.sock",
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
&payload.to_string(),
|
||||
])
|
||||
.unwrap();
|
||||
let Command::Frontend(args) = cli.command else {
|
||||
panic!("expected frontend args")
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_value(args.into_config().hf_overrides).unwrap(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hf_overrides_reject_non_object_inputs() {
|
||||
for patch in ["null", "[]", "1", r#""callable""#] {
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--hf-overrides",
|
||||
patch,
|
||||
])
|
||||
.unwrap_err();
|
||||
assert_eq!(error.kind(), clap::error::ErrorKind::ValueValidation);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frontend_args_json_accepts_supported_non_default_fields() {
|
||||
let cli = Cli::try_parse_from([
|
||||
@@ -1613,6 +1686,9 @@ fn serve_args_accept_handshake_aliases() {
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
hf_overrides: HfOverrides(
|
||||
{},
|
||||
),
|
||||
generation_config: Auto,
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
@@ -1763,6 +1839,9 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
hf_overrides: HfOverrides(
|
||||
{},
|
||||
),
|
||||
generation_config: Auto,
|
||||
served_model_name: [],
|
||||
listener_mode: BindTcp {
|
||||
@@ -1852,6 +1931,9 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
hf_overrides: HfOverrides(
|
||||
{},
|
||||
),
|
||||
generation_config: Auto,
|
||||
served_model_name: [],
|
||||
listener_mode: BindTcp {
|
||||
@@ -1963,6 +2045,9 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
},
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
hf_overrides: HfOverrides(
|
||||
{},
|
||||
),
|
||||
generation_config: Auto,
|
||||
served_model_name: [],
|
||||
listener_mode: InheritedFd {
|
||||
|
||||
@@ -235,11 +235,6 @@ pub struct EngineUnsupportedArgs {
|
||||
#[arg(long, default_missing_value = "true", num_args = 0..=1)]
|
||||
pub hf_token: Option<Unsupported>,
|
||||
|
||||
/// If a dictionary, contains arguments to be forwarded to the Hugging Face
|
||||
/// config. If a callable, it is called to update the HuggingFace config.
|
||||
#[arg(long)]
|
||||
pub hf_overrides: Option<Unsupported>,
|
||||
|
||||
/// Overrides or sets generation config. e.g. `{"temperature": 0.5}`. If
|
||||
/// used with `--generation-config auto`, the override parameters will be
|
||||
/// merged with the default config from the model. If used with
|
||||
|
||||
@@ -93,6 +93,7 @@ impl ManagedEngineArgs {
|
||||
shutdown_timeout: u64,
|
||||
handshake_port: u16,
|
||||
limit_mm_per_prompt: Option<String>,
|
||||
hf_overrides: Option<String>,
|
||||
) -> ManagedEngineConfig {
|
||||
let mut python_args = self.python_args;
|
||||
// Manually forward some args to the Python engine.
|
||||
@@ -136,6 +137,10 @@ impl ManagedEngineArgs {
|
||||
python_args.push("--limit-mm-per-prompt".to_string());
|
||||
python_args.push(limit_mm_per_prompt);
|
||||
}
|
||||
if let Some(hf_overrides) = hf_overrides {
|
||||
python_args.push("--hf-overrides".to_string());
|
||||
python_args.push(hf_overrides);
|
||||
}
|
||||
|
||||
ManagedEngineConfig {
|
||||
python: self.python,
|
||||
|
||||
@@ -61,6 +61,7 @@ async fn main() -> Result<()> {
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: args.model,
|
||||
revision: None,
|
||||
hf_overrides: Default::default(),
|
||||
generation_config: Default::default(),
|
||||
served_model_name: vec![],
|
||||
listener_mode: HttpListenerMode::BindTcp {
|
||||
|
||||
@@ -16,6 +16,7 @@ use vllm_chat::{
|
||||
ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,
|
||||
};
|
||||
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
|
||||
use vllm_text::backend::hf::HfOverrides;
|
||||
|
||||
/// Default keep-alive idle timeout (seconds); also the head-read bound
|
||||
/// when keep-alive is disabled (`0`).
|
||||
@@ -214,6 +215,8 @@ pub struct Config {
|
||||
pub model: String,
|
||||
/// Model revision on the Hugging Face Hub (branch, tag, or commit SHA).
|
||||
pub revision: Option<String>,
|
||||
/// JSON Merge Patch applied to the model config before backend initialization.
|
||||
pub hf_overrides: HfOverrides,
|
||||
/// Which generation-config sampling defaults to inherit.
|
||||
pub generation_config: GenerationConfigMode,
|
||||
/// Model name(s) exposed to clients via the OpenAI API. When non-empty,
|
||||
|
||||
@@ -99,6 +99,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
&config.model,
|
||||
LoadModelBackendsOptions {
|
||||
revision: config.revision.clone(),
|
||||
hf_overrides: config.hf_overrides.clone(),
|
||||
generation_config: config.generation_config,
|
||||
renderer: config.renderer,
|
||||
language_model_only: config.language_model_only,
|
||||
|
||||
@@ -13,6 +13,7 @@ use vllm_chat::{
|
||||
ParserSelection, RendererSelection, load_model_backends,
|
||||
};
|
||||
use vllm_text::TextRequestProcessor;
|
||||
use vllm_text::backend::hf::HfOverrides;
|
||||
|
||||
use crate::{
|
||||
HttpListenerMode, TlsConfig,
|
||||
@@ -25,6 +26,7 @@ use crate::{
|
||||
pub struct RenderConfig {
|
||||
pub model: String,
|
||||
pub revision: Option<String>,
|
||||
pub hf_overrides: HfOverrides,
|
||||
pub served_model_name: Vec<String>,
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
@@ -69,6 +71,7 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
|
||||
&config.model,
|
||||
LoadModelBackendsOptions {
|
||||
revision: config.revision.clone(),
|
||||
hf_overrides: config.hf_overrides.clone(),
|
||||
generation_config: Default::default(),
|
||||
renderer: config.renderer,
|
||||
language_model_only: true,
|
||||
@@ -153,6 +156,7 @@ mod tests {
|
||||
RenderConfig {
|
||||
model: "test-model".to_string(),
|
||||
revision: None,
|
||||
hf_overrides: Default::default(),
|
||||
served_model_name: Vec::new(),
|
||||
host: "127.0.0.1".to_string(),
|
||||
port: 8000,
|
||||
|
||||
@@ -16,6 +16,7 @@ reqwest.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_with.workspace = true
|
||||
tempfile.workspace = true
|
||||
thiserror.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
tracing.workspace = true
|
||||
@@ -28,7 +29,6 @@ vllm-tokenizer.workspace = true
|
||||
expect-test.workspace = true
|
||||
futures.workspace = true
|
||||
serial_test.workspace = true
|
||||
tempfile.workspace = true
|
||||
tokio.workspace = true
|
||||
vllm-llm = { workspace = true, features = ["test-util"] }
|
||||
vllm-tokenizer = { workspace = true, features = ["test-utils"] }
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
mod config;
|
||||
mod model_files;
|
||||
mod overrides;
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
@@ -18,6 +19,7 @@ pub use self::config::{
|
||||
load_tokenizer_config,
|
||||
};
|
||||
pub use self::model_files::{ResolvedModelFiles, TokenizerSource};
|
||||
pub use self::overrides::HfOverrides;
|
||||
use crate::backend::{GenerationConfigMode, SamplingHints, TextBackend};
|
||||
use crate::error::Result;
|
||||
|
||||
|
||||
@@ -2,11 +2,13 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use hf_hub::api::tokio::{Api, ApiBuilder, ApiRepo};
|
||||
use hf_hub::{Cache, Repo, RepoType};
|
||||
use thiserror_ext::AsReport as _;
|
||||
|
||||
use super::HfOverrides;
|
||||
use super::config::{HfTokenizerConfig, load_tokenizer_config};
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
@@ -50,9 +52,27 @@ pub struct ResolvedModelFiles {
|
||||
pub processor_config_path: Option<PathBuf>,
|
||||
pub chat_template_path: Option<PathBuf>,
|
||||
pub config_path: Option<PathBuf>,
|
||||
/// Keep a patched config alive across clones and backend ownership.
|
||||
config_temp_path: Option<Arc<tempfile::TempPath>>,
|
||||
}
|
||||
|
||||
impl ResolvedModelFiles {
|
||||
/// Materialize a merge-patched config owned by these resolved model files.
|
||||
pub fn apply_overrides(&mut self, overrides: &HfOverrides) -> Result<()> {
|
||||
if overrides.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let path = materialize_config(self.config_path.as_deref(), overrides).map_err(|error| {
|
||||
Error::Tokenizer(format!(
|
||||
"failed to materialize HF overrides: {}",
|
||||
error.as_report()
|
||||
))
|
||||
})?;
|
||||
self.config_path = Some(path.to_path_buf());
|
||||
self.config_temp_path = Some(Arc::new(path));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolve tokenizer/config files from a local model directory first when
|
||||
/// `model_id` points to one, otherwise consult the local HF cache and
|
||||
/// finally the Hub.
|
||||
@@ -84,6 +104,24 @@ impl ResolvedModelFiles {
|
||||
}
|
||||
}
|
||||
|
||||
fn materialize_config(
|
||||
path: Option<&Path>,
|
||||
overrides: &HfOverrides,
|
||||
) -> anyhow::Result<tempfile::TempPath> {
|
||||
let mut config: serde_json::Value = match path {
|
||||
Some(path) => serde_json::from_slice(&std::fs::read(path)?)?,
|
||||
None => serde_json::json!({}),
|
||||
};
|
||||
anyhow::ensure!(config.is_object(), "HF model config must be a JSON object");
|
||||
overrides.apply(&mut config);
|
||||
let file = tempfile::Builder::new()
|
||||
.prefix("vllm-rs-hf-config-")
|
||||
.suffix(".json")
|
||||
.tempfile()?;
|
||||
serde_json::to_writer(file.as_file(), &config)?;
|
||||
Ok(file.into_temp_path())
|
||||
}
|
||||
|
||||
fn resolve_local_model_files(model_dir: &Path) -> Result<ResolvedModelFiles> {
|
||||
let tokenizer_config_path = local_file_if_exists(model_dir, "tokenizer_config.json");
|
||||
let tokenizer_config = load_tokenizer_config(tokenizer_config_path.as_deref())?;
|
||||
@@ -101,6 +139,7 @@ fn resolve_local_model_files(model_dir: &Path) -> Result<ResolvedModelFiles> {
|
||||
processor_config_path: local_file_if_exists(model_dir, "processor_config.json"),
|
||||
chat_template_path: discover_chat_template_in_dir(model_dir),
|
||||
config_path: local_file_if_exists(model_dir, "config.json"),
|
||||
config_temp_path: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -164,6 +203,7 @@ async fn resolve_remote_model_files(
|
||||
processor_config_path,
|
||||
chat_template_path,
|
||||
config_path,
|
||||
config_temp_path: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -204,6 +244,7 @@ fn resolve_cached_model_files(cache: &Cache, repo: &Repo) -> Result<Option<Resol
|
||||
processor_config_path,
|
||||
chat_template_path,
|
||||
config_path,
|
||||
config_temp_path: None,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -434,6 +475,37 @@ mod tests {
|
||||
|
||||
use super::{ResolvedModelFiles, TokenizerSource};
|
||||
|
||||
#[tokio::test]
|
||||
async fn overrides_keep_configs_alive_across_clones_and_isolate_instances() {
|
||||
let source = tempdir().unwrap();
|
||||
let original = r#"{"vocab_size":10}"#;
|
||||
fs::write(source.path().join("config.json"), original).unwrap();
|
||||
fs::write(source.path().join("tokenizer.json"), "{}").unwrap();
|
||||
let mut first =
|
||||
ResolvedModelFiles::new(source.path().to_str().unwrap(), None).await.unwrap();
|
||||
let patch = serde_json::from_value(serde_json::json!({"vocab_size":20})).unwrap();
|
||||
first.apply_overrides(&patch).unwrap();
|
||||
let first_path = first.config_path.clone().unwrap();
|
||||
|
||||
let mut second = first.clone();
|
||||
second.apply_overrides(&patch).unwrap();
|
||||
let second_path = second.config_path.clone().unwrap();
|
||||
assert_ne!(first_path, second_path);
|
||||
|
||||
let retained = first.clone();
|
||||
drop(first);
|
||||
assert!(first_path.exists());
|
||||
drop(second);
|
||||
assert!(!second_path.exists());
|
||||
assert!(first_path.exists());
|
||||
drop(retained);
|
||||
assert!(!first_path.exists());
|
||||
assert_eq!(
|
||||
fs::read_to_string(source.path().join("config.json")).unwrap(),
|
||||
original
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resolved_model_files_prefers_absolute_local_model_dir() {
|
||||
let dir = tempdir().expect("create temp dir");
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
/// An object-shaped JSON Merge Patch (RFC 7396) for the model's HF config.
|
||||
/// Objects merge recursively, arrays/scalars replace, and null removes a key.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct HfOverrides(pub Map<String, Value>);
|
||||
|
||||
impl HfOverrides {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.0.is_empty()
|
||||
}
|
||||
|
||||
/// Apply the patch to an in-memory config, preserving the source file.
|
||||
pub fn apply(&self, config: &mut Value) {
|
||||
merge_object(config, &self.0);
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply RFC 7396 object-member updates at every nesting level.
|
||||
fn merge_object(target: &mut Value, patch: &Map<String, Value>) {
|
||||
if !target.is_object() {
|
||||
*target = Value::Object(Map::new());
|
||||
}
|
||||
let target = target.as_object_mut().unwrap();
|
||||
for (key, value) in patch {
|
||||
match value {
|
||||
Value::Null => {
|
||||
target.remove(key);
|
||||
}
|
||||
Value::Object(object) => {
|
||||
merge_object(target.entry(key.clone()).or_insert(Value::Null), object)
|
||||
}
|
||||
_ => {
|
||||
target.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serde_json::json;
|
||||
|
||||
use super::HfOverrides;
|
||||
|
||||
#[test]
|
||||
fn merge_patch_preserves_siblings_replaces_arrays_and_deletes_null_members() {
|
||||
let mut config = json!({
|
||||
"rope_parameters": {"rope_type": "yarn", "factor": 2},
|
||||
"text_config": {"rope_parameters": {"rope_type": "yarn", "factor": 2}},
|
||||
"architectures": ["Old", "Other"],
|
||||
"sliding_window": 4096
|
||||
});
|
||||
let patch: HfOverrides = serde_json::from_value(json!({
|
||||
"rope_parameters": {"factor": 4},
|
||||
"text_config": {"rope_parameters": {"factor": 4}},
|
||||
"architectures": ["New"],
|
||||
"sliding_window": null,
|
||||
"missing": null
|
||||
}))
|
||||
.unwrap();
|
||||
patch.apply(&mut config);
|
||||
assert_eq!(
|
||||
config,
|
||||
json!({
|
||||
"rope_parameters": {"rope_type": "yarn", "factor": 4},
|
||||
"text_config": {"rope_parameters": {"rope_type": "yarn", "factor": 4}},
|
||||
"architectures": ["New"]
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_patch_creates_objects_and_preserves_nulls_inside_replacement_arrays() {
|
||||
let mut config = json!({"a": 1, "b": [1, 2]});
|
||||
let patch: HfOverrides = serde_json::from_value(json!({
|
||||
"a": {"nested": {"removed": null}},
|
||||
"b": {},
|
||||
"c": [null, {"literal": null}]
|
||||
}))
|
||||
.unwrap();
|
||||
patch.apply(&mut config);
|
||||
assert_eq!(
|
||||
config,
|
||||
json!({"a": {"nested": {}}, "b": {}, "c": [null, {"literal": null}]})
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user