[Perf][Engram] Serialize offloaded lookups and pack host tables into huge pages (#56926)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Kimi Code <noreply@moonshot.cn>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Juntian Liu
2026-09-25 10:52:02 +08:00
committed by GitHub
co-authored by Kimi Code Claude Fable 5.1 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 48d8880d09
commit 6936e77ba4
6 changed files with 257 additions and 12 deletions
@@ -13,7 +13,7 @@ import tempfile
import threading
import time
from collections.abc import Callable, Generator, Iterable
from contextlib import contextmanager
from contextlib import contextmanager, suppress
from pathlib import Path
from typing import IO, TYPE_CHECKING, Any
@@ -844,6 +844,40 @@ def np_cache_weights_iterator(
yield name, torch.from_numpy(param)
def drop_checkpoint_cache(
model_name_or_path: str,
revision: str | None = None,
cache_dir: str | None = None,
) -> None:
checkpoint_dir: Path | None = None
if os.path.isdir(model_name_or_path):
checkpoint_dir = Path(model_name_or_path)
else:
with suppress(OSError, ValueError):
checkpoint_dir = Path(
hf_api().snapshot_download(
model_name_or_path,
revision=revision,
cache_dir=cache_dir,
local_files_only=True,
)
)
if checkpoint_dir is None:
logger.info_once(
"No local checkpoint found for %s; skipping page-cache eviction",
model_name_or_path,
)
return
for path in checkpoint_dir.glob("*.safetensors"):
try:
with path.open("rb") as file:
os.posix_fadvise(file.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
except OSError as exc:
logger.warning(
"Could not release checkpoint page cache for %s: %s", path, exc
)
def _get_checkpoints_size_bytes(files: list[str]) -> int:
"""Return the total size of the checkpoint files in bytes."""
if not files: