[Perf][Engram] Serialize offloaded lookups and pack host tables into huge pages (#56926)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Co-authored-by: Kimi Code <noreply@moonshot.cn> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Kimi Code
Claude Fable 5.1
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
48d8880d09
commit
6936e77ba4
@@ -13,7 +13,7 @@ import tempfile
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Generator, Iterable
|
||||
from contextlib import contextmanager
|
||||
from contextlib import contextmanager, suppress
|
||||
from pathlib import Path
|
||||
from typing import IO, TYPE_CHECKING, Any
|
||||
|
||||
@@ -844,6 +844,40 @@ def np_cache_weights_iterator(
|
||||
yield name, torch.from_numpy(param)
|
||||
|
||||
|
||||
def drop_checkpoint_cache(
|
||||
model_name_or_path: str,
|
||||
revision: str | None = None,
|
||||
cache_dir: str | None = None,
|
||||
) -> None:
|
||||
checkpoint_dir: Path | None = None
|
||||
if os.path.isdir(model_name_or_path):
|
||||
checkpoint_dir = Path(model_name_or_path)
|
||||
else:
|
||||
with suppress(OSError, ValueError):
|
||||
checkpoint_dir = Path(
|
||||
hf_api().snapshot_download(
|
||||
model_name_or_path,
|
||||
revision=revision,
|
||||
cache_dir=cache_dir,
|
||||
local_files_only=True,
|
||||
)
|
||||
)
|
||||
if checkpoint_dir is None:
|
||||
logger.info_once(
|
||||
"No local checkpoint found for %s; skipping page-cache eviction",
|
||||
model_name_or_path,
|
||||
)
|
||||
return
|
||||
for path in checkpoint_dir.glob("*.safetensors"):
|
||||
try:
|
||||
with path.open("rb") as file:
|
||||
os.posix_fadvise(file.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Could not release checkpoint page cache for %s: %s", path, exc
|
||||
)
|
||||
|
||||
|
||||
def _get_checkpoints_size_bytes(files: list[str]) -> int:
|
||||
"""Return the total size of the checkpoint files in bytes."""
|
||||
if not files:
|
||||
|
||||
Reference in New Issue
Block a user