[Bugfix][KV Offload] Check cgroup memory before SHM allocation (#54014)

Signed-off-by: Alex <jihui.huang@daocloud.io>
Signed-off-by: AlexHuang <alex.tech.lab@outlook.com>
Signed-off-by: Alex <alex.tech.lab@outlook.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
This commit is contained in:
AlexHuang
2026-09-16 09:17:50 +03:00
committed by GitHub
co-authored by OpenAI Codex Itay Etelis
parent e0301afb2a
commit 0f8fa53acf
7 changed files with 343 additions and 34 deletions
@@ -34,6 +34,7 @@ import vllm.envs as envs
from vllm.distributed.utils import StatelessProcessGroup, sched_yield
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.cpu_resource_utils import check_cgroup_memory_available
from vllm.utils.network_utils import (
get_ip,
get_open_zmq_inproc_path,
@@ -224,26 +225,37 @@ class SpinCondition:
SHM_PATH = "/dev/shm"
def check_shm_free_space(required_bytes: int, shm_path: str = SHM_PATH) -> None:
"""Raise if ``shm_path`` cannot fit a ``required_bytes`` shared segment.
def check_shm_free_space(
required_bytes: int,
shm_path: str = SHM_PATH,
*,
allocation_name: str = "shared-memory allocation",
) -> None:
"""Raise if SHM cannot fit a shared segment and log cgroup headroom.
Args:
required_bytes: Size of the shared-memory segment to be created.
shm_path: Mount point backing POSIX shared memory; skipped if absent.
shm_path: Mount point backing POSIX shared memory; its filesystem
check is skipped if absent.
allocation_name: Human-readable name used in errors and logs.
Raises:
RuntimeError: If ``required_bytes`` exceeds the free space.
RuntimeError: If the SHM filesystem has insufficient space.
"""
if not os.path.isdir(shm_path):
return
free_bytes = shutil.disk_usage(shm_path).free
if required_bytes <= free_bytes:
return
mib = 1 << 20
raise RuntimeError(
f"Insufficient space in {shm_path}: {required_bytes / mib:.0f} MiB "
f"required, {free_bytes / mib:.0f} MiB free. Increase {shm_path} "
"(e.g. --shm-size or --ipc=host)."
if os.path.isdir(shm_path):
free_bytes = shutil.disk_usage(shm_path).free
if required_bytes > free_bytes:
mib = 1 << 20
raise RuntimeError(
f"Insufficient space in {shm_path} for {allocation_name}: "
f"{required_bytes / mib:.0f} MiB required, "
f"{free_bytes / mib:.0f} MiB free. Increase {shm_path} "
"(e.g. --shm-size or --ipc=host)."
)
check_cgroup_memory_available(
required_bytes,
allocation_name,
)