[Rust Frontend] Honor HF revisions, offline mode, and cache directory (#56386)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -3,8 +3,8 @@
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use hf_hub::Cache;
|
||||
use hf_hub::api::tokio::{Api, ApiBuilder, ApiRepo};
|
||||
use hf_hub::{Cache, Repo, RepoType};
|
||||
use thiserror_ext::AsReport as _;
|
||||
|
||||
use super::config::{HfTokenizerConfig, load_tokenizer_config};
|
||||
@@ -56,14 +56,31 @@ impl ResolvedModelFiles {
|
||||
/// Resolve tokenizer/config files from a local model directory first when
|
||||
/// `model_id` points to one, otherwise consult the local HF cache and
|
||||
/// finally the Hub.
|
||||
pub async fn new(model_id: &str) -> Result<Self> {
|
||||
pub async fn new(model_id: &str, revision: Option<&str>) -> Result<Self> {
|
||||
if Path::new(model_id).is_dir() {
|
||||
return resolve_local_model_files(Path::new(model_id));
|
||||
}
|
||||
if let Some(files) = resolve_cached_model_files(model_id)? {
|
||||
let repo = Repo::with_revision(
|
||||
model_id.to_string(),
|
||||
RepoType::Model,
|
||||
revision.unwrap_or("main").to_string(),
|
||||
);
|
||||
// TODO: Let hf-hub read HF_HUB_CACHE after upgrading to 1.0.
|
||||
let cache = std::env::var_os("HF_HUB_CACHE")
|
||||
.map(|path| Cache::new(path.into()))
|
||||
.unwrap_or_else(Cache::from_env);
|
||||
if let Some(files) = resolve_cached_model_files(&cache, &repo)? {
|
||||
return Ok(files);
|
||||
}
|
||||
resolve_remote_model_files(model_id).await
|
||||
if is_hf_hub_offline() {
|
||||
return Err(Error::Tokenizer(format!(
|
||||
"model '{model_id}' at revision '{}' has no supported tokenizer in cache '{}' \
|
||||
and HF_HUB_OFFLINE is enabled",
|
||||
repo.revision(),
|
||||
cache.path().display(),
|
||||
)));
|
||||
}
|
||||
resolve_remote_model_files(model_id, repo, &cache).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,9 +104,13 @@ fn resolve_local_model_files(model_dir: &Path) -> Result<ResolvedModelFiles> {
|
||||
})
|
||||
}
|
||||
|
||||
async fn resolve_remote_model_files(model_id: &str) -> Result<ResolvedModelFiles> {
|
||||
let api = build_api().map_err(|error| Error::Tokenizer(error.to_report_string()))?;
|
||||
let repo = api.model(model_id.to_string());
|
||||
async fn resolve_remote_model_files(
|
||||
model_id: &str,
|
||||
model_repo: Repo,
|
||||
cache: &Cache,
|
||||
) -> Result<ResolvedModelFiles> {
|
||||
let api = build_api(cache).map_err(|error| Error::Tokenizer(error.to_report_string()))?;
|
||||
let repo = api.repo(model_repo);
|
||||
let info = repo.info().await.map_err(|error| {
|
||||
Error::Tokenizer(format!(
|
||||
"failed to fetch model '{model_id}': {}",
|
||||
@@ -146,12 +167,20 @@ async fn resolve_remote_model_files(model_id: &str) -> Result<ResolvedModelFiles
|
||||
})
|
||||
}
|
||||
|
||||
fn resolve_cached_model_files(model_id: &str) -> Result<Option<ResolvedModelFiles>> {
|
||||
let cache_repo = Cache::from_env().model(model_id.to_string());
|
||||
fn resolve_cached_model_files(cache: &Cache, repo: &Repo) -> Result<Option<ResolvedModelFiles>> {
|
||||
let cache_repo = cache.repo(repo.clone());
|
||||
// hf-hub 0.5 looks up refs even for commit SHAs; Python caches have no such ref.
|
||||
let snapshot_dir = (repo.revision().len() == 40
|
||||
&& repo.revision().bytes().all(|byte| byte.is_ascii_hexdigit()))
|
||||
.then(|| cache.path().join(repo.folder_name()).join("snapshots").join(repo.revision()));
|
||||
let get_file = |name: &str| match &snapshot_dir {
|
||||
Some(dir) => local_file_if_exists(dir, name),
|
||||
None => cache_repo.get(name),
|
||||
};
|
||||
|
||||
let tokenizer_config_path = cache_repo.get("tokenizer_config.json");
|
||||
let tokenizer_config_path = get_file("tokenizer_config.json");
|
||||
let tokenizer_config = load_tokenizer_config(tokenizer_config_path.as_deref())?;
|
||||
let tokenizer = match resolve_cached_tokenizer_source(&cache_repo, &tokenizer_config)? {
|
||||
let tokenizer = match resolve_cached_tokenizer_source(&get_file, &tokenizer_config)? {
|
||||
Some(tokenizer) => tokenizer,
|
||||
None => return Ok(None),
|
||||
};
|
||||
@@ -159,12 +188,12 @@ fn resolve_cached_model_files(model_id: &str) -> Result<Option<ResolvedModelFile
|
||||
let model_dir = tokenizer.path().parent().ok_or_else(|| {
|
||||
Error::Tokenizer("resolved tokenizer file has no parent directory".to_string())
|
||||
})?;
|
||||
let generation_config_path = cache_repo.get("generation_config.json");
|
||||
let preprocessor_config_path = cache_repo.get("preprocessor_config.json");
|
||||
let video_preprocessor_config_path = cache_repo.get("video_preprocessor_config.json");
|
||||
let processor_config_path = cache_repo.get("processor_config.json");
|
||||
let generation_config_path = get_file("generation_config.json");
|
||||
let preprocessor_config_path = get_file("preprocessor_config.json");
|
||||
let video_preprocessor_config_path = get_file("video_preprocessor_config.json");
|
||||
let processor_config_path = get_file("processor_config.json");
|
||||
let chat_template_path = discover_chat_template_in_dir(model_dir);
|
||||
let config_path = cache_repo.get("config.json");
|
||||
let config_path = get_file("config.json");
|
||||
|
||||
Ok(Some(ResolvedModelFiles {
|
||||
tokenizer,
|
||||
@@ -207,20 +236,20 @@ async fn resolve_remote_tokenizer_source(
|
||||
}
|
||||
|
||||
fn resolve_cached_tokenizer_source(
|
||||
cache_repo: &hf_hub::CacheRepo,
|
||||
get_file: &impl Fn(&str) -> Option<PathBuf>,
|
||||
tokenizer_config: &HfTokenizerConfig,
|
||||
) -> Result<Option<TokenizerSource>> {
|
||||
let tekken_path = cache_repo.get("tekken.json");
|
||||
let tekken_path = get_file("tekken.json");
|
||||
|
||||
if let Some(tekken_path) = tekken_path {
|
||||
return Ok(Some(TokenizerSource::Tekken(tekken_path)));
|
||||
}
|
||||
|
||||
let Some(tokenizer_path) = cache_repo.get("tokenizer.json").or_else(|| {
|
||||
let Some(tokenizer_path) = get_file("tokenizer.json").or_else(|| {
|
||||
// tiktoken.model is the most common name, try it first.
|
||||
cache_repo.get("tiktoken.model").or_else(|| {
|
||||
get_file("tiktoken.model").or_else(|| {
|
||||
// Scan for any *.tiktoken file in the cache snapshot directory.
|
||||
let snapshot_dir = cache_repo.get("config.json")?.parent()?.to_path_buf();
|
||||
let snapshot_dir = get_file("config.json")?.parent()?.to_path_buf();
|
||||
discover_tiktoken_in_dir(&snapshot_dir)
|
||||
})
|
||||
}) else {
|
||||
@@ -315,8 +344,18 @@ async fn download_known_file(repo: &ApiRepo, model_id: &str, filename: &str) ->
|
||||
})
|
||||
}
|
||||
|
||||
fn build_api() -> anyhow::Result<Api> {
|
||||
let mut builder = ApiBuilder::from_env().with_progress(true);
|
||||
fn is_hf_hub_offline() -> bool {
|
||||
std::env::var("HF_HUB_OFFLINE").is_ok_and(|value| {
|
||||
matches!(
|
||||
value.to_ascii_uppercase().as_str(),
|
||||
"1" | "ON" | "YES" | "TRUE"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn build_api(cache: &Cache) -> anyhow::Result<Api> {
|
||||
let mut builder =
|
||||
ApiBuilder::from_env().with_cache_dir(cache.path().clone()).with_progress(true);
|
||||
if let Ok(token) = std::env::var(HF_TOKEN_ENV)
|
||||
&& !token.is_empty()
|
||||
{
|
||||
@@ -388,6 +427,7 @@ fn discover_chat_template_in_dir(dir: &std::path::Path) -> Option<PathBuf> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use tempfile::tempdir;
|
||||
use vllm_tokenizer::{TiktokenTokenizer, Tokenizer};
|
||||
@@ -405,7 +445,7 @@ mod tests {
|
||||
.expect("write tokenizer config");
|
||||
fs::write(dir.path().join("config.json"), "{}").expect("write config");
|
||||
|
||||
let files = ResolvedModelFiles::new(dir.path().to_str().expect("utf8 path"))
|
||||
let files = ResolvedModelFiles::new(dir.path().to_str().expect("utf8 path"), None)
|
||||
.await
|
||||
.expect("resolve local model files");
|
||||
|
||||
@@ -422,10 +462,90 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_model_files_respect_branch_tag_and_commit_revisions() {
|
||||
let dir = tempdir().unwrap();
|
||||
let cache = hf_hub::Cache::new(dir.path().to_path_buf());
|
||||
let model = "test/model";
|
||||
let main_sha = "a".repeat(40);
|
||||
let pinned_sha = "b".repeat(40);
|
||||
for (revision, sha) in [("main", &main_sha), ("release/v1", &pinned_sha)] {
|
||||
let repo =
|
||||
hf_hub::Repo::with_revision(model.into(), hf_hub::RepoType::Model, revision.into());
|
||||
cache.repo(repo.clone()).create_ref(sha).unwrap();
|
||||
let snapshot = dir.path().join(repo.folder_name()).join("snapshots").join(sha);
|
||||
fs::create_dir_all(&snapshot).unwrap();
|
||||
fs::write(snapshot.join("tokenizer.json"), "{}").unwrap();
|
||||
fs::write(snapshot.join("config.json"), "{}").unwrap();
|
||||
}
|
||||
for (revision, expected_sha) in [
|
||||
("main", Some(&main_sha)),
|
||||
("release/v1", Some(&pinned_sha)),
|
||||
(pinned_sha.as_str(), Some(&pinned_sha)),
|
||||
("missing", None),
|
||||
] {
|
||||
let repo =
|
||||
hf_hub::Repo::with_revision(model.into(), hf_hub::RepoType::Model, revision.into());
|
||||
let files = super::resolve_cached_model_files(&cache, &repo).unwrap();
|
||||
match expected_sha {
|
||||
Some(sha) => {
|
||||
let snapshot = dir.path().join(repo.folder_name()).join("snapshots").join(sha);
|
||||
let files = files.unwrap();
|
||||
assert_eq!(files.tokenizer.path(), snapshot.join("tokenizer.json"));
|
||||
assert_eq!(files.config_path, Some(snapshot.join("config.json")));
|
||||
}
|
||||
None => assert!(files.is_none()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn offline_resolution_uses_hub_cache_and_reports_cache_misses() {
|
||||
const CHILD: &str = "VLLM_TEST_HF_OFFLINE_CHILD";
|
||||
if std::env::var_os(CHILD).is_some() {
|
||||
let files = ResolvedModelFiles::new("test/model", Some("release")).await.unwrap();
|
||||
let cache = PathBuf::from(std::env::var_os("HF_HUB_CACHE").unwrap());
|
||||
assert_eq!(
|
||||
files.tokenizer.path(),
|
||||
cache.join("models--test--model/snapshots/commit/tokenizer.json")
|
||||
);
|
||||
let error = ResolvedModelFiles::new("test/model", Some("missing")).await.unwrap_err();
|
||||
let message = thiserror_ext::AsReport::as_report(&error).to_string();
|
||||
assert!(message.contains("HF_HUB_OFFLINE is enabled"), "{message}");
|
||||
assert!(message.contains("revision 'missing'"), "{message}");
|
||||
return;
|
||||
}
|
||||
|
||||
// Isolate environment variables from other tests and Tokio worker threads.
|
||||
let dir = tempdir().unwrap();
|
||||
let cache = dir.path().join("hub-cache");
|
||||
let repo = cache.join("models--test--model");
|
||||
fs::create_dir_all(repo.join("refs")).unwrap();
|
||||
fs::create_dir_all(repo.join("snapshots/commit")).unwrap();
|
||||
fs::write(repo.join("refs/release"), "commit").unwrap();
|
||||
fs::write(repo.join("snapshots/commit/tokenizer.json"), "{}").unwrap();
|
||||
for value in ["1", "ON", "yes", "True"] {
|
||||
let output = std::process::Command::new(std::env::current_exe().unwrap())
|
||||
.args(["--exact", "backend::hf::model_files::tests::offline_resolution_uses_hub_cache_and_reports_cache_misses", "--nocapture"])
|
||||
.env(CHILD, "1")
|
||||
.env("HF_HOME", dir.path().join("hf-home"))
|
||||
.env("HF_HUB_CACHE", &cache)
|
||||
.env("HF_HUB_OFFLINE", value)
|
||||
.env("HF_ENDPOINT", "http://127.0.0.1:1")
|
||||
.output().unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "too slow for CI and requires network access to Hugging Face"]
|
||||
async fn tiktoken_real_kimi_k25_tokenizer_files_load_and_handle_special_tokens() {
|
||||
let files = ResolvedModelFiles::new("moonshotai/Kimi-K2.5")
|
||||
let files = ResolvedModelFiles::new("moonshotai/Kimi-K2.5", None)
|
||||
.await
|
||||
.expect("resolve real Kimi K2.5 model files");
|
||||
|
||||
|
||||
@@ -802,7 +802,8 @@ mod tests {
|
||||
#[file_serial(hf_qwen3)]
|
||||
async fn lower_text_request_uses_real_qwen_generation_defaults() {
|
||||
let model_id = "Qwen/Qwen3-0.6B";
|
||||
let files = ResolvedModelFiles::new(model_id).await.expect("resolve qwen model files");
|
||||
let files =
|
||||
ResolvedModelFiles::new(model_id, None).await.expect("resolve qwen model files");
|
||||
let backend = HfTextBackend::from_resolved_model_files(
|
||||
files,
|
||||
model_id.to_string(),
|
||||
|
||||
Reference in New Issue
Block a user