[Rust Frontend] Honor HF revisions, offline mode, and cache directory (#56386)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -110,6 +110,9 @@ pub enum BenchCommand {
|
||||
pub struct RenderArgs {
|
||||
/// Model identifier or local model directory containing tokenizer files.
|
||||
model: String,
|
||||
/// Model revision on the Hugging Face Hub (branch, tag, or commit SHA).
|
||||
#[arg(long)]
|
||||
revision: Option<String>,
|
||||
/// HTTP bind host.
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
@@ -157,6 +160,7 @@ impl RenderArgs {
|
||||
|
||||
RenderConfig {
|
||||
model: self.model,
|
||||
revision: self.revision,
|
||||
served_model_name: self.served_model_name,
|
||||
host: self.host,
|
||||
port: self.port,
|
||||
@@ -195,6 +199,11 @@ pub struct SharedRuntimeArgs {
|
||||
/// public model ID.
|
||||
pub model: String,
|
||||
|
||||
/// Model revision on the Hugging Face Hub (branch, tag, or commit SHA).
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub revision: Option<String>,
|
||||
|
||||
/// The source of generation-config sampling defaults. `"auto"` loads the
|
||||
/// model's defaults, while `"vllm"` uses vLLM's neutral defaults.
|
||||
#[arg(long, default_value_t)]
|
||||
@@ -481,6 +490,7 @@ impl SharedRuntimeArgs {
|
||||
None => CoordinatorMode::None,
|
||||
},
|
||||
model: self.model,
|
||||
revision: self.revision,
|
||||
generation_config: self.generation_config,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
|
||||
@@ -536,6 +546,7 @@ impl SharedRuntimeArgs {
|
||||
},
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: self.model,
|
||||
revision: self.revision,
|
||||
generation_config: self.generation_config,
|
||||
served_model_name: self.served_model_name,
|
||||
listener_mode,
|
||||
@@ -743,6 +754,7 @@ impl ServeArgs {
|
||||
|
||||
self.managed_engine.clone().into_config(
|
||||
self.runtime.model.clone(),
|
||||
self.runtime.revision.clone(),
|
||||
self.runtime.max_logprobs,
|
||||
profiler_config,
|
||||
reasoning_parser.as_deref(),
|
||||
|
||||
@@ -36,6 +36,8 @@ fn render_args_build_config_without_tls() {
|
||||
"vllm-rs",
|
||||
"render",
|
||||
"Qwen/Qwen2.5-0.5B-Instruct",
|
||||
"--revision",
|
||||
"release",
|
||||
"--port",
|
||||
"8080",
|
||||
"--max-model-len",
|
||||
@@ -55,6 +57,7 @@ fn render_args_build_config_without_tls() {
|
||||
let config = args.into_config();
|
||||
|
||||
assert_eq!(config.model, "Qwen/Qwen2.5-0.5B-Instruct");
|
||||
assert_eq!(config.revision.as_deref(), Some("release"));
|
||||
assert_eq!(config.host, "127.0.0.1");
|
||||
assert_eq!(config.port, 8080);
|
||||
assert_eq!(config.max_model_len, Some(32768));
|
||||
@@ -173,6 +176,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
uds: None,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
generation_config: Auto,
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
@@ -1007,6 +1011,7 @@ fn frontend_args_accept_json() {
|
||||
data_parallel_size: None,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
generation_config: Auto,
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: None,
|
||||
@@ -1125,7 +1130,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
|
||||
"--output-address",
|
||||
"ipc:///tmp/output.sock",
|
||||
"--args-json",
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","generation_config":"vllm","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_logprobs":-1,"shutdown_timeout":3}"#,
|
||||
r#"{"model_tag":"Qwen/Qwen3-0.6B","generation_config":"vllm","revision":"release","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_logprobs":-1,"shutdown_timeout":3}"#,
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
@@ -1133,6 +1138,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
|
||||
panic!("expected frontend args");
|
||||
};
|
||||
assert_eq!(args.runtime.engine_ready_timeout_secs, 42);
|
||||
assert_eq!(args.runtime.revision.as_deref(), Some("release"));
|
||||
assert_eq!(args.runtime.generation_config, GenerationConfigMode::Vllm);
|
||||
assert_eq!(
|
||||
args.runtime.tool_call_parser,
|
||||
@@ -1604,6 +1610,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
uds: None,
|
||||
runtime: SharedRuntimeArgs {
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
generation_config: Auto,
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
@@ -1752,6 +1759,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
},
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
generation_config: Auto,
|
||||
served_model_name: [],
|
||||
listener_mode: BindTcp {
|
||||
@@ -1839,6 +1847,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
},
|
||||
coordinator_mode: MaybeInProc,
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
generation_config: Auto,
|
||||
served_model_name: [],
|
||||
listener_mode: BindTcp {
|
||||
@@ -1948,6 +1957,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
address: "tcp://127.0.0.1:7000",
|
||||
},
|
||||
model: "Qwen/Qwen3-0.6B",
|
||||
revision: None,
|
||||
generation_config: Auto,
|
||||
served_model_name: [],
|
||||
listener_mode: InheritedFd {
|
||||
@@ -2087,3 +2097,18 @@ fn frontend_args_json_disables_profiling_when_profiler_type_is_null() {
|
||||
let config = args.into_config();
|
||||
assert_eq!(config.profiler, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_revision_reaches_frontend_and_managed_engine() {
|
||||
let cli =
|
||||
Cli::try_parse_from(["vllm-rs", "serve", "test/model", "--revision", "release"]).unwrap();
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
assert_eq!(
|
||||
args.to_frontend_config("tcp://localhost:1234".into()).revision.as_deref(),
|
||||
Some("release")
|
||||
);
|
||||
let engine = args.to_managed_engine_config(1234);
|
||||
assert!(engine.python_args.windows(2).any(|args| args == ["--revision", "release"]));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user