[Rust Frontend] Integrate MM video support (#47959)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-10 08:15:33 +00:00
committed by GitHub
parent 216ee58780
commit 074bdd0d99
16 changed files with 1576 additions and 521 deletions
@@ -41,6 +41,10 @@ pub struct ResolvedModelFiles {
pub tokenizer_config_path: Option<PathBuf>,
pub generation_config_path: Option<PathBuf>,
pub preprocessor_config_path: Option<PathBuf>,
/// Video-specific preprocessor config, when provided by the model repo.
pub video_preprocessor_config_path: Option<PathBuf>,
/// Combined processor config, which may embed a `video_processor` section.
pub processor_config_path: Option<PathBuf>,
pub chat_template_path: Option<PathBuf>,
pub config_path: Option<PathBuf>,
}
@@ -70,6 +74,11 @@ fn resolve_local_model_files(model_dir: &Path) -> Result<ResolvedModelFiles> {
tokenizer_config_path,
generation_config_path: local_file_if_exists(model_dir, "generation_config.json"),
preprocessor_config_path: local_file_if_exists(model_dir, "preprocessor_config.json"),
video_preprocessor_config_path: local_file_if_exists(
model_dir,
"video_preprocessor_config.json",
),
processor_config_path: local_file_if_exists(model_dir, "processor_config.json"),
chat_template_path: discover_chat_template_in_dir(model_dir),
config_path: local_file_if_exists(model_dir, "config.json"),
})
@@ -107,6 +116,10 @@ async fn resolve_remote_model_files(model_id: &str) -> Result<ResolvedModelFiles
download_if_present(&repo, model_id, &siblings, "generation_config.json").await?;
let preprocessor_config_path =
download_if_present(&repo, model_id, &siblings, "preprocessor_config.json").await?;
let video_preprocessor_config_path =
download_if_present(&repo, model_id, &siblings, "video_preprocessor_config.json").await?;
let processor_config_path =
download_if_present(&repo, model_id, &siblings, "processor_config.json").await?;
let chat_template_name = siblings
.contains("chat_template.json")
.then_some("chat_template.json")
@@ -123,6 +136,8 @@ async fn resolve_remote_model_files(model_id: &str) -> Result<ResolvedModelFiles
tokenizer_config_path,
generation_config_path,
preprocessor_config_path,
video_preprocessor_config_path,
processor_config_path,
chat_template_path,
config_path,
})
@@ -143,6 +158,8 @@ fn resolve_cached_model_files(model_id: &str) -> Result<Option<ResolvedModelFile
})?;
let generation_config_path = cache_repo.get("generation_config.json");
let preprocessor_config_path = cache_repo.get("preprocessor_config.json");
let video_preprocessor_config_path = cache_repo.get("video_preprocessor_config.json");
let processor_config_path = cache_repo.get("processor_config.json");
let chat_template_path = discover_chat_template_in_dir(model_dir);
let config_path = cache_repo.get("config.json");
@@ -151,6 +168,8 @@ fn resolve_cached_model_files(model_id: &str) -> Result<Option<ResolvedModelFile
tokenizer_config_path,
generation_config_path,
preprocessor_config_path,
video_preprocessor_config_path,
processor_config_path,
chat_template_path,
config_path,
}))