[Rust Frontend] Support HF config overrides --hf-overrides (#56931)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -61,6 +61,7 @@ async fn main() -> Result<()> {
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: args.model,
|
||||
revision: None,
|
||||
hf_overrides: Default::default(),
|
||||
generation_config: Default::default(),
|
||||
served_model_name: vec![],
|
||||
listener_mode: HttpListenerMode::BindTcp {
|
||||
|
||||
@@ -16,6 +16,7 @@ use vllm_chat::{
|
||||
ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,
|
||||
};
|
||||
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
|
||||
use vllm_text::backend::hf::HfOverrides;
|
||||
|
||||
/// Default keep-alive idle timeout (seconds); also the head-read bound
|
||||
/// when keep-alive is disabled (`0`).
|
||||
@@ -214,6 +215,8 @@ pub struct Config {
|
||||
pub model: String,
|
||||
/// Model revision on the Hugging Face Hub (branch, tag, or commit SHA).
|
||||
pub revision: Option<String>,
|
||||
/// JSON Merge Patch applied to the model config before backend initialization.
|
||||
pub hf_overrides: HfOverrides,
|
||||
/// Which generation-config sampling defaults to inherit.
|
||||
pub generation_config: GenerationConfigMode,
|
||||
/// Model name(s) exposed to clients via the OpenAI API. When non-empty,
|
||||
|
||||
@@ -99,6 +99,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
&config.model,
|
||||
LoadModelBackendsOptions {
|
||||
revision: config.revision.clone(),
|
||||
hf_overrides: config.hf_overrides.clone(),
|
||||
generation_config: config.generation_config,
|
||||
renderer: config.renderer,
|
||||
language_model_only: config.language_model_only,
|
||||
|
||||
@@ -13,6 +13,7 @@ use vllm_chat::{
|
||||
ParserSelection, RendererSelection, load_model_backends,
|
||||
};
|
||||
use vllm_text::TextRequestProcessor;
|
||||
use vllm_text::backend::hf::HfOverrides;
|
||||
|
||||
use crate::{
|
||||
HttpListenerMode, TlsConfig,
|
||||
@@ -25,6 +26,7 @@ use crate::{
|
||||
pub struct RenderConfig {
|
||||
pub model: String,
|
||||
pub revision: Option<String>,
|
||||
pub hf_overrides: HfOverrides,
|
||||
pub served_model_name: Vec<String>,
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
@@ -69,6 +71,7 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
|
||||
&config.model,
|
||||
LoadModelBackendsOptions {
|
||||
revision: config.revision.clone(),
|
||||
hf_overrides: config.hf_overrides.clone(),
|
||||
generation_config: Default::default(),
|
||||
renderer: config.renderer,
|
||||
language_model_only: true,
|
||||
@@ -153,6 +156,7 @@ mod tests {
|
||||
RenderConfig {
|
||||
model: "test-model".to_string(),
|
||||
revision: None,
|
||||
hf_overrides: Default::default(),
|
||||
served_model_name: Vec::new(),
|
||||
host: "127.0.0.1".to_string(),
|
||||
port: 8000,
|
||||
|
||||
Reference in New Issue
Block a user