[DS V4.1][Engram] Support async prefetch for offloaded engram lookups and engram DP sharding (#56512)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: ywang96 <>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Juntian Liu
2026-09-13 02:44:23 -07:00
committed by GitHub
co-authored by Roger Wang ywang96 <> Woosuk Kwon Codex
parent e7a3963339
commit d2d649e674
12 changed files with 1821 additions and 195 deletions
+80
View File
@@ -32,6 +32,7 @@ from collections.abc import Callable
from contextlib import AbstractContextManager, contextmanager, nullcontext
from dataclasses import dataclass
from datetime import timedelta
from math import gcd
from multiprocessing import shared_memory
from typing import TYPE_CHECKING, Any, Protocol
from unittest.mock import patch
@@ -1558,6 +1559,23 @@ def get_etp_group() -> GroupCoordinator:
return _ETP
_ENGRAM_DP: GroupCoordinator | None = None
def get_engram_dp_group() -> GroupCoordinator | None:
"""Return the DP replicas that share one engram embedding table.
None when every replica holds a full (TP-sharded) copy, which is the case
for models without engram layers and for replicas that span nodes.
"""
return _ENGRAM_DP
def get_engram_dp_size() -> int:
"""Number of DP replicas one engram embedding table is sharded over."""
return _ENGRAM_DP.world_size if _ENGRAM_DP is not None else 1
_DCP: GroupCoordinator | None = None
@@ -1933,6 +1951,29 @@ def _elastic_join_warmup_ctx() -> AbstractContextManager[Any]:
return nullcontext()
def _engram_dp_shard_size(
world_size: int,
data_parallel_size: int,
replica_size: int,
) -> int:
"""Size of DP shards aligned with complete replicas and node boundaries."""
node_count = get_node_count()
replicas_per_node, remainder = divmod(world_size, node_count * replica_size)
if remainder:
return 1
# Shards must divide both the DP group and each node's replica count.
shard_size = gcd(data_parallel_size, replicas_per_node)
if shard_size == 1 or node_count == 1:
return shard_size
ranks_per_node = replicas_per_node * replica_size
for start in range(0, world_size, ranks_per_node):
same_node = in_the_same_node_as(get_world_group().cpu_group, start)
node_ranks = [rank for rank, local in enumerate(same_node) if local]
if node_ranks != list(range(start, start + ranks_per_node)):
return 1
return shard_size
def initialize_model_parallel(
tensor_model_parallel_size: int = 1,
pipeline_model_parallel_size: int = 1,
@@ -2055,6 +2096,40 @@ def initialize_model_parallel(
group_name="etp",
)
# Build the node-local DP group used to shard or share an Engram table.
global _ENGRAM_DP
assert _ENGRAM_DP is None, "engram data parallel group is already initialized"
engram_dp_size = 1
engram_config = config.engram_config
if (
engram_config is not None
and config.model_config is not None
and config.model_config.architecture == "DeepseekV41ForCausalLM"
and not enable_elastic_ep
):
engram_dp_size = _engram_dp_shard_size(
world_size,
data_parallel_size,
tensor_model_parallel_size
* pipeline_model_parallel_size
* prefill_context_model_parallel_size,
)
if engram_dp_size > 1:
group_ranks = (
all_ranks.permute(0, 2, 3, 4, 1).reshape(-1, engram_dp_size).unbind(0)
)
_ENGRAM_DP = init_model_parallel_group(
[x.tolist() for x in group_ranks],
get_world_group().local_rank,
backend,
group_name="edp",
)
logger.info_once(
"Engram node-local DP group size: %d (TP=%d)",
engram_dp_size,
tensor_model_parallel_size,
)
# Build the DCP model-parallel groups.
global _DCP
assert _DCP is None, "decode context model parallel group is already initialized"
@@ -2304,6 +2379,11 @@ def destroy_model_parallel():
_TP.destroy()
_TP = None
global _ENGRAM_DP
if _ENGRAM_DP:
_ENGRAM_DP.destroy()
_ENGRAM_DP = None
global _DCP
if _DCP:
_DCP.destroy()