[Rust Frontend] Honor HF revisions, offline mode, and cache directory (#56386)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Bugen Zhao
2026-09-12 03:39:17 +00:00
committed by GitHub
co-authored by Codex
parent 5f3e4b7447
commit 485421b1c3
11 changed files with 201 additions and 28 deletions
+145 -25
View File
@@ -3,8 +3,8 @@
use std::path::{Path, PathBuf};
use hf_hub::Cache;
use hf_hub::api::tokio::{Api, ApiBuilder, ApiRepo};
use hf_hub::{Cache, Repo, RepoType};
use thiserror_ext::AsReport as _;
use super::config::{HfTokenizerConfig, load_tokenizer_config};
@@ -56,14 +56,31 @@ impl ResolvedModelFiles {
/// Resolve tokenizer/config files from a local model directory first when
/// `model_id` points to one, otherwise consult the local HF cache and
/// finally the Hub.
pub async fn new(model_id: &str) -> Result<Self> {
pub async fn new(model_id: &str, revision: Option<&str>) -> Result<Self> {
if Path::new(model_id).is_dir() {
return resolve_local_model_files(Path::new(model_id));
}
if let Some(files) = resolve_cached_model_files(model_id)? {
let repo = Repo::with_revision(
model_id.to_string(),
RepoType::Model,
revision.unwrap_or("main").to_string(),
);
// TODO: Let hf-hub read HF_HUB_CACHE after upgrading to 1.0.
let cache = std::env::var_os("HF_HUB_CACHE")
.map(|path| Cache::new(path.into()))
.unwrap_or_else(Cache::from_env);
if let Some(files) = resolve_cached_model_files(&cache, &repo)? {
return Ok(files);
}
resolve_remote_model_files(model_id).await
if is_hf_hub_offline() {
return Err(Error::Tokenizer(format!(
"model '{model_id}' at revision '{}' has no supported tokenizer in cache '{}' \
and HF_HUB_OFFLINE is enabled",
repo.revision(),
cache.path().display(),
)));
}
resolve_remote_model_files(model_id, repo, &cache).await
}
}
@@ -87,9 +104,13 @@ fn resolve_local_model_files(model_dir: &Path) -> Result<ResolvedModelFiles> {
})
}
async fn resolve_remote_model_files(model_id: &str) -> Result<ResolvedModelFiles> {
let api = build_api().map_err(|error| Error::Tokenizer(error.to_report_string()))?;
let repo = api.model(model_id.to_string());
async fn resolve_remote_model_files(
model_id: &str,
model_repo: Repo,
cache: &Cache,
) -> Result<ResolvedModelFiles> {
let api = build_api(cache).map_err(|error| Error::Tokenizer(error.to_report_string()))?;
let repo = api.repo(model_repo);
let info = repo.info().await.map_err(|error| {
Error::Tokenizer(format!(
"failed to fetch model '{model_id}': {}",
@@ -146,12 +167,20 @@ async fn resolve_remote_model_files(model_id: &str) -> Result<ResolvedModelFiles
})
}
fn resolve_cached_model_files(model_id: &str) -> Result<Option<ResolvedModelFiles>> {
let cache_repo = Cache::from_env().model(model_id.to_string());
fn resolve_cached_model_files(cache: &Cache, repo: &Repo) -> Result<Option<ResolvedModelFiles>> {
let cache_repo = cache.repo(repo.clone());
// hf-hub 0.5 looks up refs even for commit SHAs; Python caches have no such ref.
let snapshot_dir = (repo.revision().len() == 40
&& repo.revision().bytes().all(|byte| byte.is_ascii_hexdigit()))
.then(|| cache.path().join(repo.folder_name()).join("snapshots").join(repo.revision()));
let get_file = |name: &str| match &snapshot_dir {
Some(dir) => local_file_if_exists(dir, name),
None => cache_repo.get(name),
};
let tokenizer_config_path = cache_repo.get("tokenizer_config.json");
let tokenizer_config_path = get_file("tokenizer_config.json");
let tokenizer_config = load_tokenizer_config(tokenizer_config_path.as_deref())?;
let tokenizer = match resolve_cached_tokenizer_source(&cache_repo, &tokenizer_config)? {
let tokenizer = match resolve_cached_tokenizer_source(&get_file, &tokenizer_config)? {
Some(tokenizer) => tokenizer,
None => return Ok(None),
};
@@ -159,12 +188,12 @@ fn resolve_cached_model_files(model_id: &str) -> Result<Option<ResolvedModelFile
let model_dir = tokenizer.path().parent().ok_or_else(|| {
Error::Tokenizer("resolved tokenizer file has no parent directory".to_string())
})?;
let generation_config_path = cache_repo.get("generation_config.json");
let preprocessor_config_path = cache_repo.get("preprocessor_config.json");
let video_preprocessor_config_path = cache_repo.get("video_preprocessor_config.json");
let processor_config_path = cache_repo.get("processor_config.json");
let generation_config_path = get_file("generation_config.json");
let preprocessor_config_path = get_file("preprocessor_config.json");
let video_preprocessor_config_path = get_file("video_preprocessor_config.json");
let processor_config_path = get_file("processor_config.json");
let chat_template_path = discover_chat_template_in_dir(model_dir);
let config_path = cache_repo.get("config.json");
let config_path = get_file("config.json");
Ok(Some(ResolvedModelFiles {
tokenizer,
@@ -207,20 +236,20 @@ async fn resolve_remote_tokenizer_source(
}
fn resolve_cached_tokenizer_source(
cache_repo: &hf_hub::CacheRepo,
get_file: &impl Fn(&str) -> Option<PathBuf>,
tokenizer_config: &HfTokenizerConfig,
) -> Result<Option<TokenizerSource>> {
let tekken_path = cache_repo.get("tekken.json");
let tekken_path = get_file("tekken.json");
if let Some(tekken_path) = tekken_path {
return Ok(Some(TokenizerSource::Tekken(tekken_path)));
}
let Some(tokenizer_path) = cache_repo.get("tokenizer.json").or_else(|| {
let Some(tokenizer_path) = get_file("tokenizer.json").or_else(|| {
// tiktoken.model is the most common name, try it first.
cache_repo.get("tiktoken.model").or_else(|| {
get_file("tiktoken.model").or_else(|| {
// Scan for any *.tiktoken file in the cache snapshot directory.
let snapshot_dir = cache_repo.get("config.json")?.parent()?.to_path_buf();
let snapshot_dir = get_file("config.json")?.parent()?.to_path_buf();
discover_tiktoken_in_dir(&snapshot_dir)
})
}) else {
@@ -315,8 +344,18 @@ async fn download_known_file(repo: &ApiRepo, model_id: &str, filename: &str) ->
})
}
fn build_api() -> anyhow::Result<Api> {
let mut builder = ApiBuilder::from_env().with_progress(true);
fn is_hf_hub_offline() -> bool {
std::env::var("HF_HUB_OFFLINE").is_ok_and(|value| {
matches!(
value.to_ascii_uppercase().as_str(),
"1" | "ON" | "YES" | "TRUE"
)
})
}
fn build_api(cache: &Cache) -> anyhow::Result<Api> {
let mut builder =
ApiBuilder::from_env().with_cache_dir(cache.path().clone()).with_progress(true);
if let Ok(token) = std::env::var(HF_TOKEN_ENV)
&& !token.is_empty()
{
@@ -388,6 +427,7 @@ fn discover_chat_template_in_dir(dir: &std::path::Path) -> Option<PathBuf> {
#[cfg(test)]
mod tests {
use std::fs;
use std::path::PathBuf;
use tempfile::tempdir;
use vllm_tokenizer::{TiktokenTokenizer, Tokenizer};
@@ -405,7 +445,7 @@ mod tests {
.expect("write tokenizer config");
fs::write(dir.path().join("config.json"), "{}").expect("write config");
let files = ResolvedModelFiles::new(dir.path().to_str().expect("utf8 path"))
let files = ResolvedModelFiles::new(dir.path().to_str().expect("utf8 path"), None)
.await
.expect("resolve local model files");
@@ -422,10 +462,90 @@ mod tests {
);
}
#[test]
fn cached_model_files_respect_branch_tag_and_commit_revisions() {
let dir = tempdir().unwrap();
let cache = hf_hub::Cache::new(dir.path().to_path_buf());
let model = "test/model";
let main_sha = "a".repeat(40);
let pinned_sha = "b".repeat(40);
for (revision, sha) in [("main", &main_sha), ("release/v1", &pinned_sha)] {
let repo =
hf_hub::Repo::with_revision(model.into(), hf_hub::RepoType::Model, revision.into());
cache.repo(repo.clone()).create_ref(sha).unwrap();
let snapshot = dir.path().join(repo.folder_name()).join("snapshots").join(sha);
fs::create_dir_all(&snapshot).unwrap();
fs::write(snapshot.join("tokenizer.json"), "{}").unwrap();
fs::write(snapshot.join("config.json"), "{}").unwrap();
}
for (revision, expected_sha) in [
("main", Some(&main_sha)),
("release/v1", Some(&pinned_sha)),
(pinned_sha.as_str(), Some(&pinned_sha)),
("missing", None),
] {
let repo =
hf_hub::Repo::with_revision(model.into(), hf_hub::RepoType::Model, revision.into());
let files = super::resolve_cached_model_files(&cache, &repo).unwrap();
match expected_sha {
Some(sha) => {
let snapshot = dir.path().join(repo.folder_name()).join("snapshots").join(sha);
let files = files.unwrap();
assert_eq!(files.tokenizer.path(), snapshot.join("tokenizer.json"));
assert_eq!(files.config_path, Some(snapshot.join("config.json")));
}
None => assert!(files.is_none()),
}
}
}
#[tokio::test]
async fn offline_resolution_uses_hub_cache_and_reports_cache_misses() {
const CHILD: &str = "VLLM_TEST_HF_OFFLINE_CHILD";
if std::env::var_os(CHILD).is_some() {
let files = ResolvedModelFiles::new("test/model", Some("release")).await.unwrap();
let cache = PathBuf::from(std::env::var_os("HF_HUB_CACHE").unwrap());
assert_eq!(
files.tokenizer.path(),
cache.join("models--test--model/snapshots/commit/tokenizer.json")
);
let error = ResolvedModelFiles::new("test/model", Some("missing")).await.unwrap_err();
let message = thiserror_ext::AsReport::as_report(&error).to_string();
assert!(message.contains("HF_HUB_OFFLINE is enabled"), "{message}");
assert!(message.contains("revision 'missing'"), "{message}");
return;
}
// Isolate environment variables from other tests and Tokio worker threads.
let dir = tempdir().unwrap();
let cache = dir.path().join("hub-cache");
let repo = cache.join("models--test--model");
fs::create_dir_all(repo.join("refs")).unwrap();
fs::create_dir_all(repo.join("snapshots/commit")).unwrap();
fs::write(repo.join("refs/release"), "commit").unwrap();
fs::write(repo.join("snapshots/commit/tokenizer.json"), "{}").unwrap();
for value in ["1", "ON", "yes", "True"] {
let output = std::process::Command::new(std::env::current_exe().unwrap())
.args(["--exact", "backend::hf::model_files::tests::offline_resolution_uses_hub_cache_and_reports_cache_misses", "--nocapture"])
.env(CHILD, "1")
.env("HF_HOME", dir.path().join("hf-home"))
.env("HF_HUB_CACHE", &cache)
.env("HF_HUB_OFFLINE", value)
.env("HF_ENDPOINT", "http://127.0.0.1:1")
.output().unwrap();
assert!(
output.status.success(),
"{}\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
}
#[tokio::test]
#[ignore = "too slow for CI and requires network access to Hugging Face"]
async fn tiktoken_real_kimi_k25_tokenizer_files_load_and_handle_special_tokens() {
let files = ResolvedModelFiles::new("moonshotai/Kimi-K2.5")
let files = ResolvedModelFiles::new("moonshotai/Kimi-K2.5", None)
.await
.expect("resolve real Kimi K2.5 model files");
+2 -1
View File
@@ -802,7 +802,8 @@ mod tests {
#[file_serial(hf_qwen3)]
async fn lower_text_request_uses_real_qwen_generation_defaults() {
let model_id = "Qwen/Qwen3-0.6B";
let files = ResolvedModelFiles::new(model_id).await.expect("resolve qwen model files");
let files =
ResolvedModelFiles::new(model_id, None).await.expect("resolve qwen model files");
let backend = HfTextBackend::from_resolved_model_files(
files,
model_id.to_string(),