[Rust Frontend] Honor HF revisions, offline mode, and cache directory (#56386)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -60,6 +60,7 @@ async fn main() -> Result<()> {
|
||||
},
|
||||
coordinator_mode: CoordinatorMode::MaybeInProc,
|
||||
model: args.model,
|
||||
revision: None,
|
||||
generation_config: Default::default(),
|
||||
served_model_name: vec![],
|
||||
listener_mode: HttpListenerMode::BindTcp {
|
||||
|
||||
@@ -210,6 +210,8 @@ pub struct Config {
|
||||
pub coordinator_mode: CoordinatorMode,
|
||||
/// Backend model identifier used for engine-core loading.
|
||||
pub model: String,
|
||||
/// Model revision on the Hugging Face Hub (branch, tag, or commit SHA).
|
||||
pub revision: Option<String>,
|
||||
/// Which generation-config sampling defaults to inherit.
|
||||
pub generation_config: GenerationConfigMode,
|
||||
/// Model name(s) exposed to clients via the OpenAI API. When non-empty,
|
||||
|
||||
@@ -98,6 +98,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
let loaded = load_model_backends(
|
||||
&config.model,
|
||||
LoadModelBackendsOptions {
|
||||
revision: config.revision.clone(),
|
||||
generation_config: config.generation_config,
|
||||
renderer: config.renderer,
|
||||
language_model_only: config.language_model_only,
|
||||
|
||||
@@ -24,6 +24,7 @@ use crate::{
|
||||
#[derive(Debug)]
|
||||
pub struct RenderConfig {
|
||||
pub model: String,
|
||||
pub revision: Option<String>,
|
||||
pub served_model_name: Vec<String>,
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
@@ -67,6 +68,7 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
|
||||
let loaded = load_model_backends(
|
||||
&config.model,
|
||||
LoadModelBackendsOptions {
|
||||
revision: config.revision.clone(),
|
||||
generation_config: Default::default(),
|
||||
renderer: config.renderer,
|
||||
language_model_only: true,
|
||||
@@ -150,6 +152,7 @@ mod tests {
|
||||
let error = serve_render(
|
||||
RenderConfig {
|
||||
model: "test-model".to_string(),
|
||||
revision: None,
|
||||
served_model_name: Vec::new(),
|
||||
host: "127.0.0.1".to_string(),
|
||||
port: 8000,
|
||||
|
||||
Reference in New Issue
Block a user