[Bugfix][KV Offload] Check cgroup memory before SHM allocation (#54014)

Signed-off-by: Alex <jihui.huang@daocloud.io>
Signed-off-by: AlexHuang <alex.tech.lab@outlook.com>
Signed-off-by: Alex <alex.tech.lab@outlook.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
This commit is contained in:
AlexHuang
2026-09-16 09:17:50 +03:00
committed by GitHub
co-authored by OpenAI Codex Itay Etelis
parent e0301afb2a
commit 0f8fa53acf
7 changed files with 343 additions and 34 deletions
+32 -3
View File
@@ -694,14 +694,43 @@ def test_check_shm_free_space_raises_when_insufficient(tmp_path):
def test_check_shm_free_space_passes_when_sufficient(tmp_path):
with mock.patch.object(
shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(512 << 20)
with (
mock.patch.object(
shm_broadcast.shutil,
"disk_usage",
return_value=_fake_disk_usage(512 << 20),
),
mock.patch.object(shm_broadcast, "check_cgroup_memory_available"),
):
check_shm_free_space(240 << 20, shm_path=str(tmp_path))
def test_check_shm_free_space_skipped_when_path_missing(tmp_path):
check_shm_free_space(1 << 60, shm_path=str(tmp_path / "does-not-exist"))
with mock.patch.object(shm_broadcast, "check_cgroup_memory_available"):
check_shm_free_space(1 << 60, shm_path=str(tmp_path / "does-not-exist"))
def test_check_shm_free_space_checks_cgroup(tmp_path):
with (
mock.patch.object(
shm_broadcast.shutil,
"disk_usage",
return_value=_fake_disk_usage(512 << 20),
),
mock.patch.object(
shm_broadcast, "check_cgroup_memory_available"
) as check_cgroup,
):
check_shm_free_space(
240 << 20,
shm_path=str(tmp_path),
allocation_name="SHM mmap",
)
check_cgroup.assert_called_once_with(
240 << 20,
"SHM mmap",
)
def test_shm_ring_buffer_creation_checks_free_space():
+189
View File
@@ -0,0 +1,189 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Cgroup memory readers and SHM allocation preflight checks."""
import sys
from io import StringIO
from unittest import mock
import pytest
from vllm.utils import cpu_resource_utils as cru
from vllm.utils.mem_constants import GiB_bytes
_V2_LIMIT_PATH = "/sys/fs/cgroup/memory.max"
_V2_USAGE_PATH = "/sys/fs/cgroup/memory.current"
_V1_LIMIT_PATH = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
_V1_USAGE_PATH = "/sys/fs/cgroup/memory/memory.usage_in_bytes"
@pytest.fixture(autouse=True)
def _clear_cgroup_cache():
cru.get_cgroup_memory_limit.cache_clear()
yield
cru.get_cgroup_memory_limit.cache_clear()
def _stub_files(monkeypatch, files: dict):
"""Stub ``open()`` for a fixed set of paths; ``None`` -> OSError."""
real_open = open
def fake_open(path, *args, **kwargs):
if path in files:
content = files[path]
if content is None:
raise OSError(f"no such file: {path}")
return StringIO(content)
return real_open(path, *args, **kwargs)
monkeypatch.setattr("builtins.open", fake_open)
def test_cgroup_v2_limit_and_usage(monkeypatch):
monkeypatch.setattr(sys, "platform", "linux")
_stub_files(
monkeypatch,
{
_V2_LIMIT_PATH: f"{20 * GiB_bytes}\n",
_V2_USAGE_PATH: f"{5 * GiB_bytes}\n",
},
)
assert cru.get_cgroup_memory_limit() == 20 * GiB_bytes
assert cru.get_cgroup_memory_usage() == 5 * GiB_bytes
def test_cgroup_v2_unlimited_falls_back_to_v1(monkeypatch):
monkeypatch.setattr(sys, "platform", "linux")
_stub_files(
monkeypatch,
{
_V2_LIMIT_PATH: "max\n",
_V1_LIMIT_PATH: f"{8 * GiB_bytes}\n",
_V1_USAGE_PATH: f"{1 * GiB_bytes}\n",
},
)
assert cru.get_cgroup_memory_limit() == 8 * GiB_bytes
assert cru.get_cgroup_memory_usage() == 1 * GiB_bytes
def test_cgroup_v1_limit_and_usage(monkeypatch):
monkeypatch.setattr(sys, "platform", "linux")
_stub_files(
monkeypatch,
{
_V2_LIMIT_PATH: None,
_V1_LIMIT_PATH: f"{8 * GiB_bytes}\n",
_V1_USAGE_PATH: f"{2 * GiB_bytes}\n",
},
)
assert cru.get_cgroup_memory_limit() == 8 * GiB_bytes
assert cru.get_cgroup_memory_usage() == 2 * GiB_bytes
def test_cgroup_v1_unlimited_sentinel_is_ignored(monkeypatch):
"""An unlimited cgroup v1 sentinel must not be treated as a real limit."""
monkeypatch.setattr(sys, "platform", "linux")
_stub_files(
monkeypatch,
{_V2_LIMIT_PATH: None, _V1_LIMIT_PATH: f"{(1 << 63) - 1}\n"},
)
assert cru.get_cgroup_memory_limit() is None
assert cru.get_cgroup_memory_usage() is None
def test_cgroup_no_limit_files_present(monkeypatch):
monkeypatch.setattr(sys, "platform", "linux")
_stub_files(monkeypatch, {_V2_LIMIT_PATH: None, _V1_LIMIT_PATH: None})
assert cru.get_cgroup_memory_limit() is None
assert cru.get_cgroup_memory_usage() is None
def test_cgroup_skipped_on_non_linux(monkeypatch):
monkeypatch.setattr(sys, "platform", "darwin")
_stub_files(monkeypatch, {_V2_LIMIT_PATH: f"{1 * GiB_bytes}\n"})
assert cru.get_cgroup_memory_limit() is None
assert cru.get_cgroup_memory_usage() is None
def test_cgroup_limit_is_cached_until_explicitly_cleared(monkeypatch):
monkeypatch.setattr(sys, "platform", "linux")
files = {
_V2_LIMIT_PATH: f"{1 * GiB_bytes}\n",
_V2_USAGE_PATH: f"{512 << 20}\n",
}
_stub_files(monkeypatch, files)
assert cru.get_cgroup_memory_limit() == 1 * GiB_bytes
files[_V2_LIMIT_PATH] = f"{2 * GiB_bytes}\n"
assert cru.get_cgroup_memory_limit() == 1 * GiB_bytes
cru.get_cgroup_memory_limit.cache_clear()
assert cru.get_cgroup_memory_limit() == 2 * GiB_bytes
def test_cgroup_usage_is_read_without_cache(monkeypatch):
monkeypatch.setattr(sys, "platform", "linux")
files = {
_V2_LIMIT_PATH: f"{1 * GiB_bytes}\n",
_V2_USAGE_PATH: f"{512 << 20}\n",
}
_stub_files(monkeypatch, files)
assert cru.get_cgroup_memory_usage() == 512 << 20
files[_V2_USAGE_PATH] = f"{768 << 20}\n"
assert cru.get_cgroup_memory_usage() == 768 << 20
def test_check_cgroup_memory_available_warns_on_low_headroom(monkeypatch):
monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: 1 << 30)
monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: 512 << 20)
with (
mock.patch.object(cru.logger, "debug") as log_debug,
mock.patch.object(cru.logger, "warning") as log_warning,
):
cru.check_cgroup_memory_available(600 << 20, "mmap")
log_debug.assert_not_called()
log_warning.assert_called_once()
assert "current headroom is below" in log_warning.call_args.args[-1]
def test_check_cgroup_memory_available_logs_success_at_debug(monkeypatch):
monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: 1 << 30)
monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: 512 << 20)
with (
mock.patch.object(cru.logger, "debug") as log_debug,
mock.patch.object(cru.logger, "warning") as log_warning,
):
cru.check_cgroup_memory_available(256 << 20, "mmap")
log_debug.assert_called_once()
log_warning.assert_not_called()
@pytest.mark.parametrize(
("limit", "usage"),
[(None, 512 << 20), (1 << 30, None)],
)
def test_check_cgroup_memory_available_skips_without_snapshot(
monkeypatch, limit, usage
):
monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: limit)
monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: usage)
with (
mock.patch.object(cru.logger, "debug") as log_debug,
mock.patch.object(cru.logger, "warning") as log_warning,
):
cru.check_cgroup_memory_available(1 << 60, "mmap")
log_debug.assert_not_called()
log_warning.assert_not_called()
@@ -906,12 +906,13 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
monkeypatch.setattr(region.os, "open", mock_open)
monkeypatch.setattr(region.os, "unlink", mock_unlink)
monkeypatch.setattr(region.os, "close", mock_close)
mock_check = MagicMock(
side_effect=RuntimeError("Insufficient space in /dev/shm: 30 GB required.")
)
monkeypatch.setattr(
region,
"check_shm_free_space",
lambda *a, **kw: (_ for _ in ()).throw(
RuntimeError("Insufficient space in /dev/shm: 30 GB required.")
),
mock_check,
)
with pytest.raises(RuntimeError, match="Insufficient space"):
@@ -925,6 +926,10 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
mock_unlink.assert_called_once_with(mmap_path)
mock_close.assert_called_once_with(9999)
mock_check.assert_called_once_with(
4 * PAGE_SIZE,
allocation_name="CPU KV offload shared region in /dev/shm",
)
def test_ftruncate_failure_cleans_up_creator(monkeypatch):
@@ -34,6 +34,7 @@ import vllm.envs as envs
from vllm.distributed.utils import StatelessProcessGroup, sched_yield
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.cpu_resource_utils import check_cgroup_memory_available
from vllm.utils.network_utils import (
get_ip,
get_open_zmq_inproc_path,
@@ -224,26 +225,37 @@ class SpinCondition:
SHM_PATH = "/dev/shm"
def check_shm_free_space(required_bytes: int, shm_path: str = SHM_PATH) -> None:
"""Raise if ``shm_path`` cannot fit a ``required_bytes`` shared segment.
def check_shm_free_space(
required_bytes: int,
shm_path: str = SHM_PATH,
*,
allocation_name: str = "shared-memory allocation",
) -> None:
"""Raise if SHM cannot fit a shared segment and log cgroup headroom.
Args:
required_bytes: Size of the shared-memory segment to be created.
shm_path: Mount point backing POSIX shared memory; skipped if absent.
shm_path: Mount point backing POSIX shared memory; its filesystem
check is skipped if absent.
allocation_name: Human-readable name used in errors and logs.
Raises:
RuntimeError: If ``required_bytes`` exceeds the free space.
RuntimeError: If the SHM filesystem has insufficient space.
"""
if not os.path.isdir(shm_path):
return
free_bytes = shutil.disk_usage(shm_path).free
if required_bytes <= free_bytes:
return
mib = 1 << 20
raise RuntimeError(
f"Insufficient space in {shm_path}: {required_bytes / mib:.0f} MiB "
f"required, {free_bytes / mib:.0f} MiB free. Increase {shm_path} "
"(e.g. --shm-size or --ipc=host)."
if os.path.isdir(shm_path):
free_bytes = shutil.disk_usage(shm_path).free
if required_bytes > free_bytes:
mib = 1 << 20
raise RuntimeError(
f"Insufficient space in {shm_path} for {allocation_name}: "
f"{required_bytes / mib:.0f} MiB required, "
f"{free_bytes / mib:.0f} MiB free. Increase {shm_path} "
"(e.g. --shm-size or --ipc=host)."
)
check_cgroup_memory_available(
required_bytes,
allocation_name,
)
@@ -727,11 +727,15 @@ def _get_available_ram_bytes() -> int:
host_available = psutil.virtual_memory().available
from vllm.utils.cpu_resource_utils import get_cgroup_memory_limit
from vllm.utils.cpu_resource_utils import (
get_cgroup_memory_limit,
get_cgroup_memory_usage,
)
cgroup_limit, cgroup_usage = get_cgroup_memory_limit()
cgroup_limit = get_cgroup_memory_limit()
if cgroup_limit is None:
return host_available
cgroup_usage = get_cgroup_memory_usage()
cgroup_available = (
cgroup_limit if cgroup_usage is None else max(0, cgroup_limit - cgroup_usage)
)
+78 -11
View File
@@ -12,8 +12,12 @@ from functools import cache
import psutil
import regex as re
from vllm.logger import init_logger
DEVICE_CONTROL_ENV_VAR = "CPU_VISIBLE_MEMORY_NODES"
logger = init_logger(__name__)
@dataclass
class LogicalCPUInfo:
@@ -63,21 +67,20 @@ def _read_int_file(path: str) -> int | None:
@cache
def get_cgroup_memory_limit() -> tuple[int | None, int | None]:
"""Return (limit, usage) in bytes from cgroup, or (None, None).
def get_cgroup_memory_limit() -> int | None:
"""Return the cgroup memory limit in bytes, or None.
Supports both cgroup v2 (unified) and v1. Returns (None, None) when
Supports both cgroup v2 (unified) and v1. Returns None when
not running under a constrained cgroup (e.g. bare metal, or limit
reported as `max`/an unrealistically large value).
"""
if sys.platform != "linux":
return None, None
return None
# cgroup v2 unified hierarchy
v2_limit = _read_int_file("/sys/fs/cgroup/memory.max")
if v2_limit is not None:
v2_usage = _read_int_file("/sys/fs/cgroup/memory.current")
return v2_limit, v2_usage
return v2_limit
# cgroup v1
v1_limit = _read_int_file("/sys/fs/cgroup/memory/memory.limit_in_bytes")
@@ -85,11 +88,74 @@ def get_cgroup_memory_limit() -> tuple[int | None, int | None]:
# cgroup v1 reports a huge sentinel (close to PAGE_COUNTER_MAX)
# when unlimited. Treat absurdly large values as "no limit".
if v1_limit >= (1 << 62):
return None, None
v1_usage = _read_int_file("/sys/fs/cgroup/memory/memory.usage_in_bytes")
return v1_limit, v1_usage
return None
return v1_limit
return None, None
return None
def get_cgroup_memory_usage() -> int | None:
"""Return the current cgroup memory usage in bytes, or None.
The usage value is intentionally read on every call because cgroup
memory usage changes while the process is running.
"""
if sys.platform != "linux":
return None
# cgroup v2 unified hierarchy
if _read_int_file("/sys/fs/cgroup/memory.max") is not None:
return _read_int_file("/sys/fs/cgroup/memory.current")
# cgroup v1
v1_limit = _read_int_file("/sys/fs/cgroup/memory/memory.limit_in_bytes")
if v1_limit is not None and v1_limit < (1 << 62):
return _read_int_file("/sys/fs/cgroup/memory/memory.usage_in_bytes")
return None
def check_cgroup_memory_available(
required_bytes: int,
allocation_name: str,
) -> None:
"""Log cgroup memory headroom for an upcoming allocation.
Args:
required_bytes: Bytes required by the allocation.
allocation_name: Human-readable name used in log messages.
Low headroom logs a warning, but does not reject the allocation because
cgroup usage can include reclaimable memory. If the cgroup limit or usage
cannot be read, the check is skipped.
"""
cgroup_limit = get_cgroup_memory_limit()
cgroup_usage = get_cgroup_memory_usage()
if cgroup_limit is None or cgroup_usage is None:
return
cgroup_available = max(0, cgroup_limit - cgroup_usage)
mib = 1 << 20
remaining_bytes = cgroup_available - required_bytes
log_fn = logger.debug if remaining_bytes >= 0 else logger.warning
status = (
"current headroom meets the requested allocation"
if remaining_bytes >= 0
else "current headroom is below the requested allocation; allocation "
"will still be attempted because cgroup usage may be reclaimable"
)
log_fn(
"Cgroup memory preflight for %s: %.0f MiB required, %.0f MiB current "
"usage, %.0f MiB available under %.0f MiB limit, %.0f MiB remaining "
"after allocation based on current usage; %s.",
allocation_name,
required_bytes / mib,
cgroup_usage / mib,
cgroup_available / mib,
cgroup_limit / mib,
remaining_bytes / mib,
status,
)
def get_memory_affinity(pid: int = 0) -> list[int]:
@@ -161,7 +227,8 @@ def get_memory_node_info(node_id: int = 0) -> MemoryNodeInfo:
# would be applied to host RAM instead of the pod's limit. cgroup
# does not expose per-NUMA-node limits, so we just clamp the totals
# against the pod-wide limit here.
cgroup_limit, cgroup_usage = get_cgroup_memory_limit()
cgroup_limit = get_cgroup_memory_limit()
cgroup_usage = get_cgroup_memory_usage()
if cgroup_limit is not None and cgroup_limit < total_memory:
total_memory = cgroup_limit
cgroup_available = cgroup_limit - (cgroup_usage or 0)
@@ -135,7 +135,10 @@ class SharedOffloadRegion:
self._creator = True
if creator_memory_check is not None:
creator_memory_check(self.total_size_bytes)
check_shm_free_space(self.total_size_bytes)
check_shm_free_space(
self.total_size_bytes,
allocation_name="CPU KV offload shared region in /dev/shm",
)
os.ftruncate(self.fd, self.total_size_bytes)
logger.info(
"Created mmap file %s (%.2f GB)",