[Model][Engram] Share host tables across co-located DP replicas by default (#57651)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Juntian Liu
2026-09-21 08:26:56 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 zjy0516
parent 0ff0477ff9
commit 76ffb0d374
7 changed files with 68 additions and 22 deletions
+23 -1
View File
@@ -3,6 +3,7 @@
"""NVIDIA Engram DP sharding, shared host storage, and asynchronous prefetch."""
import mmap
import shutil
import tempfile
import weakref
from contextlib import ExitStack
@@ -89,6 +90,27 @@ def gather_engram_hashes(
return dp_group.all_gather(hash_ids, dim=0)
def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool:
"""Whether co-located DP replicas exist and /dev/shm can hold the full tables."""
if get_engram_dp_size() == 1:
logger.warning_once(
"Engram DP replicas are not co-located on one node; "
"storing the offloaded tables per rank instead of sharing them."
)
return False
num_bytes = sum(layout.num_embeddings) * (
layout.head_dim + layout.head_dim // block_size
)
if shutil.disk_usage("/dev/shm").total < num_bytes:
logger.warning_once(
"Sharing Engram tables across DP replicas needs %.1f GiB of /dev/shm "
"(--shm-size or --ipc=host); sharding them across replicas instead.",
num_bytes / 1024**3,
)
return False
return True
class DPSharedEngramStorage:
"""Registered host weights shared by a node-local DP group with one writer."""
@@ -362,7 +384,7 @@ class Engram(BaseEngram):
layout.head_dim,
tuple(size for order in layout.primes[layer_hash_index] for size in order),
cpu_offload=engram_config.cpu_offload,
dp_shared_memory=engram_config.dp_shared_memory,
dp_shared_memory=bool(engram_config.dp_shared_memory),
)
def _init_staging(self, max_tokens: int, head_dim: int) -> None:
+8 -1
View File
@@ -88,7 +88,7 @@ from vllm.v1.worker.ubatching import dbo_current_ubatch_id
from ..common.engram import EngramLayout, NgramHashState
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
from .engram import Engram, gather_engram_hashes
from .engram import Engram, can_share_engram_tables, gather_engram_hashes
from .ops.mhc import (
MHC_OVERLAP_MAX_TOKENS,
mhc_pre_delayed_overlap,
@@ -589,6 +589,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
self.embed_tokens = PPMissingLayer()
self.engram_layout = EngramLayout.from_config(config)
engram_config = vllm_config.engram_config
if (
self.engram_layout is not None
and engram_config is not None
and engram_config.dp_shared_memory
):
engram_config.dp_shared_memory = can_share_engram_tables(self.engram_layout)
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,