[Perf][Engram] Serialize offloaded lookups and pack host tables into huge pages (#56926)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Kimi Code <noreply@moonshot.cn>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Juntian Liu
2026-09-25 10:52:02 +08:00
committed by GitHub
co-authored by Kimi Code Claude Fable 5.1 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 48d8880d09
commit 6936e77ba4
6 changed files with 257 additions and 12 deletions
+9
View File
@@ -50,10 +50,19 @@ class EngramConfig:
the other settings allow it, falling back to per-replica tables when DP
replicas are not co-located on one node or /dev/shm cannot hold them."""
use_thp: bool = False
"""Back private CPU-offloaded tables with transparent huge pages (best
effort, falls back to ordinary pinned pages). Prefaulting the tables at
startup takes longer. Requires cpu_offload without dp_shared_memory."""
@model_validator(mode="after")
def _validate_shared_memory(self) -> Self:
if self.dp_shared_memory and not self.cpu_offload:
raise ValueError("dp_shared_memory requires cpu_offload=True")
if self.use_thp and (not self.cpu_offload or self.dp_shared_memory):
raise ValueError(
"use_thp requires cpu_offload=True and dp_shared_memory=False"
)
return self
def verify_model_config(self, model_config: "ModelConfig | None") -> None:
@@ -13,7 +13,7 @@ import tempfile
import threading
import time
from collections.abc import Callable, Generator, Iterable
from contextlib import contextmanager
from contextlib import contextmanager, suppress
from pathlib import Path
from typing import IO, TYPE_CHECKING, Any
@@ -844,6 +844,40 @@ def np_cache_weights_iterator(
yield name, torch.from_numpy(param)
def drop_checkpoint_cache(
model_name_or_path: str,
revision: str | None = None,
cache_dir: str | None = None,
) -> None:
checkpoint_dir: Path | None = None
if os.path.isdir(model_name_or_path):
checkpoint_dir = Path(model_name_or_path)
else:
with suppress(OSError, ValueError):
checkpoint_dir = Path(
hf_api().snapshot_download(
model_name_or_path,
revision=revision,
cache_dir=cache_dir,
local_files_only=True,
)
)
if checkpoint_dir is None:
logger.info_once(
"No local checkpoint found for %s; skipping page-cache eviction",
model_name_or_path,
)
return
for path in checkpoint_dir.glob("*.safetensors"):
try:
with path.open("rb") as file:
os.posix_fadvise(file.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
except OSError as exc:
logger.warning(
"Could not release checkpoint page cache for %s: %s", path, exc
)
def _get_checkpoints_size_bytes(files: list[str]) -> int:
"""Return the total size of the checkpoint files in bytes."""
if not files:
+85 -5
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NVIDIA Engram DP sharding, shared host storage, and asynchronous prefetch."""
import ctypes
import mmap
import os
import shutil
@@ -47,6 +48,41 @@ from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
logger = init_logger(__name__)
def _allocate_huge_page_storage(num_bytes: int) -> torch.Tensor | None:
"""Register prefaulted huge pages, or return None for pinned-memory fallback."""
try:
mapping = mmap.mmap(-1, num_bytes, flags=mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS)
mapping.madvise(mmap.MADV_HUGEPAGE)
except OSError as exc:
logger.warning(
"Engram huge-page allocation failed (%s); using pinned memory.", exc
)
return None
owner = np.frombuffer(mapping, dtype=np.uint8)
# Fault the pages in before CUDA pins them, so they can be huge.
owner[:: mmap.PAGESIZE] = 0
tensor = torch.from_numpy(owner)
pointer = tensor.data_ptr()
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
if result.value != 0:
logger.warning(
"Engram cudaHostRegister failed (%s); using pinned memory.", result
)
return None
# Tensor views retain owner; its finalizer retains mapping until unregister.
finalizer = weakref.finalize(
owner, DPSharedEngramStorage._unregister, mapping, pointer
)
finalizer.atexit = False # type: ignore[misc]
if not tensor.is_pinned():
logger.warning(
"CUDA did not recognize Engram registration; using pinned memory."
)
return None
return tensor
def engram_head_shard_rank() -> int:
"""This rank's slot among the hash-head shards of one engram table.
@@ -241,9 +277,12 @@ class ParallelEngramEmbedding(BaseParallelEngramEmbedding):
block_size: int = 32,
cpu_offload: bool = False,
dp_shared_memory: bool = False,
use_thp: bool = False,
) -> None:
self.cpu_offload = cpu_offload
self.dp_shared_memory = dp_shared_memory
self.use_thp = use_thp
self._packed: torch.Tensor | None = None
self.dp_size = get_engram_dp_size()
if dp_shared_memory:
if not cpu_offload:
@@ -292,6 +331,17 @@ class ParallelEngramEmbedding(BaseParallelEngramEmbedding):
return storage.weight, storage.weight_scale_inv
if not self.cpu_offload:
return super()._allocate_weights()
if self.use_thp:
weight_bytes = self.part_num_embeddings * self.dim
packed = _allocate_huge_page_storage(
weight_bytes + weight_bytes // self.block_size
)
if packed is not None:
self._packed = packed
return (
packed[:weight_bytes].view(torch.float8_e4m3fn).view(-1, self.dim),
packed[weight_bytes:].view(-1, self.dim // self.block_size),
)
# Model initialization may be inside a CUDA device context.
return (
torch.empty(
@@ -310,6 +360,19 @@ class ParallelEngramEmbedding(BaseParallelEngramEmbedding):
),
)
def collapse_huge_pages(self) -> None:
"""Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small."""
if self._packed is None:
return
addr, num_bytes = self._packed.data_ptr(), self._packed.nbytes
libc = ctypes.CDLL(None, use_errno=True)
libc.madvise.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int)
if libc.madvise(addr, num_bytes, 25) != 0:
logger.warning(
"Engram MADV_COLLAPSE failed; keeping existing pages: %s",
os.strerror(ctypes.get_errno()),
)
def _storage(self) -> tuple[torch.Tensor, torch.Tensor]:
if self._shared_memory is not None:
return self._shared_memory.get_views(self.weight, self.weight_scale_inv)
@@ -365,6 +428,15 @@ class Engram(BaseEngram):
"""NVIDIA Engram with asynchronous offload and node-local DP lookup."""
_prefetch_stream: torch.cuda.Stream | None = None
_prefetch_done: torch.cuda.Event | None = None
def __init__(
self, *args, prefetch_stream: torch.cuda.Stream | None = None, **kwargs
) -> None:
# Layers sharing one stream serialize their offloaded lookups, so they
# take turns instead of jointly starving decoder compute of SMs.
self._prefetch_stream = prefetch_stream
super().__init__(*args, **kwargs)
def _create_embedding(
self, layout: EngramLayout, layer_hash_index: int
@@ -377,12 +449,17 @@ class Engram(BaseEngram):
tuple(size for order in layout.primes[layer_hash_index] for size in order),
cpu_offload=engram_config.cpu_offload,
dp_shared_memory=bool(engram_config.dp_shared_memory),
use_thp=engram_config.use_thp,
)
def _init_staging(self, max_tokens: int, head_dim: int) -> None:
super()._init_staging(max_tokens * self.embed_tokens.dp_size, head_dim)
if self.embed_tokens.cpu_offload:
if not self.embed_tokens.cpu_offload:
self._prefetch_stream = None
return
if self._prefetch_stream is None:
self._prefetch_stream = torch.cuda.Stream(device=self.staged_rows.device)
self._prefetch_done = torch.cuda.Event()
def prepare_embeddings(self, hash_ids: torch.Tensor) -> None:
"""Prefetch local shared rows or the DP group's gathered hash IDs."""
@@ -402,14 +479,17 @@ class Engram(BaseEngram):
hash_ids.record_stream(stream)
with torch.cuda.stream(stream):
self.embed_tokens.lookup(hash_ids, rows, background=True)
assert self._prefetch_done is not None
self._prefetch_done.record(stream)
@eager_break_during_capture
def _finish_prefetch(self, stream: torch.cuda.Stream) -> None:
torch.cuda.current_stream().wait_stream(stream)
def _finish_prefetch(self, event: torch.cuda.Event) -> None:
# Wait for this layer's lookup only, not for later layers on the stream.
torch.cuda.current_stream().wait_event(event)
def _ready_rows(self, num_tokens: int) -> torch.Tensor:
if self._prefetch_stream is not None:
self._finish_prefetch(self._prefetch_stream)
if self._prefetch_done is not None:
self._finish_prefetch(self._prefetch_done)
if self.embed_tokens.dp_size > 1:
slot = engram_gathered_num_tokens()
staged = self.staged_rows[: slot * self.embed_tokens.dp_size]
+38 -1
View File
@@ -40,7 +40,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
drop_checkpoint_cache,
)
from vllm.model_executor.models.interfaces import (
EagleModelMixin,
MixtureOfExperts,
@@ -226,6 +229,7 @@ class DeepseekV4DecoderLayer(nn.Module):
aux_stream_list: list[torch.cuda.Stream] | None = None,
candidate_block_buffer: torch.Tensor | None = None,
engram_layout: EngramLayout | None = None,
engram_prefetch_stream: torch.cuda.Stream | None = None,
run_gemm_rs: bool = False,
mhc_stream: torch.cuda.Stream | None = None,
fuse_mhc_all_reduce: bool = False,
@@ -249,6 +253,7 @@ class DeepseekV4DecoderLayer(nn.Module):
engram_layout.layer_ids.index(layer_id),
use_sequence_parallel=self.use_sequence_parallel,
prefix=f"{prefix}.engram",
prefetch_stream=engram_prefetch_stream,
)
self.rms_norm_eps = config.rms_norm_eps
@@ -621,6 +626,14 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
self.engram_layout = EngramLayout.from_config(config)
engram_config = vllm_config.engram_config
self.vllm_config = vllm_config
engram_prefetch_stream = (
torch.cuda.Stream()
if self.engram_layout is not None
and engram_config is not None
and engram_config.cpu_offload
else None
)
if (
self.engram_layout is not None
and engram_config is not None
@@ -628,6 +641,15 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
):
engram_config.dp_shared_memory = can_share_engram_tables(self.engram_layout)
if self.engram_layout is not None and engram_config and engram_config.use_thp:
# Release old checkpoint cache before allocating the Engram host tables.
model_config = vllm_config.model_config
drop_checkpoint_cache(
model_config.model_weights or model_config.model,
revision=model_config.revision,
cache_dir=vllm_config.load_config.download_dir,
)
# GEMM-RS uses NCCL symmetric-memory multicast, which requires all TP
# ranks to belong to one NVLink domain. Collective: run before layers.
self.run_gemm_rs = maybe_init_gemm_rs(vllm_config, self.use_sequence_parallel)
@@ -641,6 +663,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
aux_stream_list=aux_stream_list,
candidate_block_buffer=self.candidate_block_buffer,
engram_layout=self.engram_layout,
engram_prefetch_stream=engram_prefetch_stream,
run_gemm_rs=self.run_gemm_rs,
mhc_stream=mhc_stream,
fuse_mhc_all_reduce=self.fuse_mhc_all_reduce,
@@ -1329,6 +1352,20 @@ class DeepseekV41LLMForCausalLM(
loader = AutoWeightsLoader(self)
loaded_params = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
self.process_weights_after_loading()
config = self.model.vllm_config
if config.engram_config and config.engram_config.use_thp:
# Loading weights refills the file cache; release it before MADV_COLLAPSE.
model_config = config.model_config
drop_checkpoint_cache(
model_config.model_weights or model_config.model,
revision=model_config.revision,
cache_dir=config.load_config.download_dir,
)
for layer in islice(
self.model.layers, self.model.start_layer, self.model.end_layer
):
if layer.engram is not None:
layer.engram.embed_tokens.collapse_huge_pages()
return loaded_params
def process_weights_after_loading(self) -> None: