[Model][Engram] Share host tables across co-located DP replicas by default (#57651)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
zjy0516
parent
0ff0477ff9
commit
76ffb0d374
@@ -3,6 +3,7 @@
|
||||
"""NVIDIA Engram DP sharding, shared host storage, and asynchronous prefetch."""
|
||||
|
||||
import mmap
|
||||
import shutil
|
||||
import tempfile
|
||||
import weakref
|
||||
from contextlib import ExitStack
|
||||
@@ -89,6 +90,27 @@ def gather_engram_hashes(
|
||||
return dp_group.all_gather(hash_ids, dim=0)
|
||||
|
||||
|
||||
def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool:
|
||||
"""Whether co-located DP replicas exist and /dev/shm can hold the full tables."""
|
||||
if get_engram_dp_size() == 1:
|
||||
logger.warning_once(
|
||||
"Engram DP replicas are not co-located on one node; "
|
||||
"storing the offloaded tables per rank instead of sharing them."
|
||||
)
|
||||
return False
|
||||
num_bytes = sum(layout.num_embeddings) * (
|
||||
layout.head_dim + layout.head_dim // block_size
|
||||
)
|
||||
if shutil.disk_usage("/dev/shm").total < num_bytes:
|
||||
logger.warning_once(
|
||||
"Sharing Engram tables across DP replicas needs %.1f GiB of /dev/shm "
|
||||
"(--shm-size or --ipc=host); sharding them across replicas instead.",
|
||||
num_bytes / 1024**3,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class DPSharedEngramStorage:
|
||||
"""Registered host weights shared by a node-local DP group with one writer."""
|
||||
|
||||
@@ -362,7 +384,7 @@ class Engram(BaseEngram):
|
||||
layout.head_dim,
|
||||
tuple(size for order in layout.primes[layer_hash_index] for size in order),
|
||||
cpu_offload=engram_config.cpu_offload,
|
||||
dp_shared_memory=engram_config.dp_shared_memory,
|
||||
dp_shared_memory=bool(engram_config.dp_shared_memory),
|
||||
)
|
||||
|
||||
def _init_staging(self, max_tokens: int, head_dim: int) -> None:
|
||||
|
||||
@@ -88,7 +88,7 @@ from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
|
||||
from ..common.engram import EngramLayout, NgramHashState
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
|
||||
from .engram import Engram, gather_engram_hashes
|
||||
from .engram import Engram, can_share_engram_tables, gather_engram_hashes
|
||||
from .ops.mhc import (
|
||||
MHC_OVERLAP_MAX_TOKENS,
|
||||
mhc_pre_delayed_overlap,
|
||||
@@ -589,6 +589,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.engram_layout = EngramLayout.from_config(config)
|
||||
engram_config = vllm_config.engram_config
|
||||
if (
|
||||
self.engram_layout is not None
|
||||
and engram_config is not None
|
||||
and engram_config.dp_shared_memory
|
||||
):
|
||||
engram_config.dp_shared_memory = can_share_engram_tables(self.engram_layout)
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
|
||||
Reference in New Issue
Block a user