[Bugfix][Engram] Fall back when /dev/shm is absent before sharing tables (#57914)

Signed-off-by: Juntian Liu <juntianl@inferact.ai>
Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Juntian Liu
2026-09-23 00:19:09 +08:00
committed by GitHub
co-authored by Claude Fable 5.1 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 066ee91197
commit a6131b0e65
+50 -58
View File
@@ -3,6 +3,7 @@
"""NVIDIA Engram DP sharding, shared host storage, and asynchronous prefetch.""" """NVIDIA Engram DP sharding, shared host storage, and asynchronous prefetch."""
import mmap import mmap
import os
import shutil import shutil
import tempfile import tempfile
import weakref import weakref
@@ -20,6 +21,10 @@ from vllm.distributed import (
get_tensor_model_parallel_rank, get_tensor_model_parallel_rank,
tensor_model_parallel_all_gather, tensor_model_parallel_all_gather,
) )
from vllm.distributed.device_communicators.shm_broadcast import (
SHM_PATH,
check_shm_free_space,
)
from vllm.distributed.parallel_state import GroupCoordinator from vllm.distributed.parallel_state import GroupCoordinator
from vllm.forward_context import get_forward_context from vllm.forward_context import get_forward_context
from vllm.logger import init_logger from vllm.logger import init_logger
@@ -92,6 +97,13 @@ def gather_engram_hashes(
def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool: def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool:
"""Whether co-located DP replicas exist and /dev/shm can hold the full tables.""" """Whether co-located DP replicas exist and /dev/shm can hold the full tables."""
if not os.path.isdir(SHM_PATH):
logger.warning_once(
"%s is not mounted; storing the offloaded Engram tables per rank "
"instead of sharing them across DP replicas.",
SHM_PATH,
)
return False
if get_engram_dp_size() == 1: if get_engram_dp_size() == 1:
logger.warning_once( logger.warning_once(
"Engram DP replicas are not co-located on one node; " "Engram DP replicas are not co-located on one node; "
@@ -101,11 +113,12 @@ def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool:
num_bytes = sum(layout.num_embeddings) * ( num_bytes = sum(layout.num_embeddings) * (
layout.head_dim + layout.head_dim // block_size layout.head_dim + layout.head_dim // block_size
) )
if shutil.disk_usage("/dev/shm").total < num_bytes: if shutil.disk_usage(SHM_PATH).total < num_bytes:
logger.warning_once( logger.warning_once(
"Sharing Engram tables across DP replicas needs %.1f GiB of /dev/shm " "Sharing Engram tables across DP replicas needs %.1f GiB of %s "
"(--shm-size or --ipc=host); sharding them across replicas instead.", "(--shm-size or --ipc=host); sharding them across replicas instead.",
num_bytes / 1024**3, num_bytes / 1024**3,
SHM_PATH,
) )
return False return False
return True return True
@@ -126,20 +139,14 @@ class DPSharedEngramStorage:
def _allocate(self, num_bytes: int) -> torch.Tensor: def _allocate(self, num_bytes: int) -> torch.Tensor:
"""Map and register one physical allocation across a node-local DP group.""" """Map and register one physical allocation across a node-local DP group."""
from vllm.distributed.device_communicators.shm_broadcast import (
check_shm_free_space,
)
group = self.group group = self.group
with ExitStack() as stack: with ExitStack() as stack:
path, error = None, None path = error = None
if group.rank_in_group == 0: if group.rank_in_group == 0:
try: try:
check_shm_free_space(num_bytes) check_shm_free_space(num_bytes)
backing_file = stack.enter_context( backing_file = stack.enter_context(
tempfile.NamedTemporaryFile( tempfile.NamedTemporaryFile(prefix="vllm_engram_", dir=SHM_PATH)
prefix="vllm_engram_", dir="/dev/shm"
)
) )
backing_file.truncate(num_bytes) backing_file.truncate(num_bytes)
path = backing_file.name path = backing_file.name
@@ -151,57 +158,42 @@ class DPSharedEngramStorage:
"Engram shared-memory creation failed on EDP rank 0: " + error "Engram shared-memory creation failed on EDP rank 0: " + error
) )
mapping = owner = tensor = finalizer = None mapping = owner = tensor = None
stage = "open"
try: try:
try: with open(path, "r+b") as file:
with open(path, "r+b") as file: mapping = mmap.mmap(file.fileno(), num_bytes, flags=mmap.MAP_SHARED)
stage = "mmap" owner = np.frombuffer(mapping, dtype=np.uint8)
mapping = mmap.mmap( pointer = owner.ctypes.data
file.fileno(), num_bytes, flags=mmap.MAP_SHARED tensor = torch.from_numpy(owner)
) result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
stage = "cudaHostRegister" if result.value != 0:
owner = np.frombuffer(mapping, dtype=np.uint8) raise RuntimeError(f"cudaHostRegister failed: {result}")
pointer = owner.ctypes.data finalizer = weakref.finalize(owner, self._unregister, mapping, pointer)
tensor = torch.from_numpy(owner) finalizer.atexit = False # type: ignore[misc]
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0) # The UVA helper otherwise allocates a private pinned copy.
if result.value != 0: if not tensor.is_pinned():
raise RuntimeError(f"cudaHostRegister failed: {result}")
finalizer = weakref.finalize(
owner, self._unregister, mapping, pointer
)
finalizer.atexit = False # type: ignore[misc]
# The UVA helper otherwise allocates a private pinned copy.
if not tensor.is_pinned():
raise RuntimeError(
"CUDA did not recognize the shared Engram registration"
)
except Exception as exc:
error = f"{stage}: {type(exc).__name__}: {exc}"
errors: list[str | None] = [None] * group.world_size
# Also fences peer mappings before the leader unlinks the file.
torch.distributed.all_gather_object(
errors, error, group=group.cpu_group
)
failures = "; ".join(
f"EDP rank {rank}: {error}"
for rank, error in enumerate(errors)
if error is not None
)
if failures:
raise RuntimeError( raise RuntimeError(
"Engram shared-memory initialization failed: " + failures "cudaHostRegister did not pin the shared Engram mapping"
) )
assert tensor is not None except Exception as exc:
return tensor error = f"{type(exc).__name__}: {exc}"
except Exception:
if finalizer is not None: # Also fences peer mappings before the leader unlinks the file.
finalizer() errors: list[str | None] = [None] * group.world_size
tensor = owner = None torch.distributed.all_gather_object(errors, error, group=group.cpu_group)
if mapping is not None: failures = "; ".join(
mapping.close() f"EDP rank {rank}: {error}"
raise for rank, error in enumerate(errors)
if error is not None
)
if failures:
# Dropping the owner runs the finalizer; the mapping then unmaps.
del tensor, owner, mapping
raise RuntimeError(
"Engram shared-memory initialization failed: " + failures
)
assert tensor is not None
return tensor
@staticmethod @staticmethod
def _unregister(mapping: mmap.mmap, pointer: int) -> None: def _unregister(mapping: mmap.mmap, pointer: int) -> None: