[Bugfix][KV Offload] Check cgroup memory before SHM allocation (#54014)
Signed-off-by: Alex <jihui.huang@daocloud.io> Signed-off-by: AlexHuang <alex.tech.lab@outlook.com> Signed-off-by: Alex <alex.tech.lab@outlook.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Itay Etelis
parent
e0301afb2a
commit
0f8fa53acf
@@ -694,14 +694,43 @@ def test_check_shm_free_space_raises_when_insufficient(tmp_path):
|
||||
|
||||
|
||||
def test_check_shm_free_space_passes_when_sufficient(tmp_path):
|
||||
with mock.patch.object(
|
||||
shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(512 << 20)
|
||||
with (
|
||||
mock.patch.object(
|
||||
shm_broadcast.shutil,
|
||||
"disk_usage",
|
||||
return_value=_fake_disk_usage(512 << 20),
|
||||
),
|
||||
mock.patch.object(shm_broadcast, "check_cgroup_memory_available"),
|
||||
):
|
||||
check_shm_free_space(240 << 20, shm_path=str(tmp_path))
|
||||
|
||||
|
||||
def test_check_shm_free_space_skipped_when_path_missing(tmp_path):
|
||||
check_shm_free_space(1 << 60, shm_path=str(tmp_path / "does-not-exist"))
|
||||
with mock.patch.object(shm_broadcast, "check_cgroup_memory_available"):
|
||||
check_shm_free_space(1 << 60, shm_path=str(tmp_path / "does-not-exist"))
|
||||
|
||||
|
||||
def test_check_shm_free_space_checks_cgroup(tmp_path):
|
||||
with (
|
||||
mock.patch.object(
|
||||
shm_broadcast.shutil,
|
||||
"disk_usage",
|
||||
return_value=_fake_disk_usage(512 << 20),
|
||||
),
|
||||
mock.patch.object(
|
||||
shm_broadcast, "check_cgroup_memory_available"
|
||||
) as check_cgroup,
|
||||
):
|
||||
check_shm_free_space(
|
||||
240 << 20,
|
||||
shm_path=str(tmp_path),
|
||||
allocation_name="SHM mmap",
|
||||
)
|
||||
|
||||
check_cgroup.assert_called_once_with(
|
||||
240 << 20,
|
||||
"SHM mmap",
|
||||
)
|
||||
|
||||
|
||||
def test_shm_ring_buffer_creation_checks_free_space():
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Cgroup memory readers and SHM allocation preflight checks."""
|
||||
|
||||
import sys
|
||||
from io import StringIO
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.utils import cpu_resource_utils as cru
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
_V2_LIMIT_PATH = "/sys/fs/cgroup/memory.max"
|
||||
_V2_USAGE_PATH = "/sys/fs/cgroup/memory.current"
|
||||
_V1_LIMIT_PATH = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
|
||||
_V1_USAGE_PATH = "/sys/fs/cgroup/memory/memory.usage_in_bytes"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_cgroup_cache():
|
||||
cru.get_cgroup_memory_limit.cache_clear()
|
||||
yield
|
||||
cru.get_cgroup_memory_limit.cache_clear()
|
||||
|
||||
|
||||
def _stub_files(monkeypatch, files: dict):
|
||||
"""Stub ``open()`` for a fixed set of paths; ``None`` -> OSError."""
|
||||
real_open = open
|
||||
|
||||
def fake_open(path, *args, **kwargs):
|
||||
if path in files:
|
||||
content = files[path]
|
||||
if content is None:
|
||||
raise OSError(f"no such file: {path}")
|
||||
return StringIO(content)
|
||||
return real_open(path, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr("builtins.open", fake_open)
|
||||
|
||||
|
||||
def test_cgroup_v2_limit_and_usage(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
_stub_files(
|
||||
monkeypatch,
|
||||
{
|
||||
_V2_LIMIT_PATH: f"{20 * GiB_bytes}\n",
|
||||
_V2_USAGE_PATH: f"{5 * GiB_bytes}\n",
|
||||
},
|
||||
)
|
||||
|
||||
assert cru.get_cgroup_memory_limit() == 20 * GiB_bytes
|
||||
assert cru.get_cgroup_memory_usage() == 5 * GiB_bytes
|
||||
|
||||
|
||||
def test_cgroup_v2_unlimited_falls_back_to_v1(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
_stub_files(
|
||||
monkeypatch,
|
||||
{
|
||||
_V2_LIMIT_PATH: "max\n",
|
||||
_V1_LIMIT_PATH: f"{8 * GiB_bytes}\n",
|
||||
_V1_USAGE_PATH: f"{1 * GiB_bytes}\n",
|
||||
},
|
||||
)
|
||||
|
||||
assert cru.get_cgroup_memory_limit() == 8 * GiB_bytes
|
||||
assert cru.get_cgroup_memory_usage() == 1 * GiB_bytes
|
||||
|
||||
|
||||
def test_cgroup_v1_limit_and_usage(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
_stub_files(
|
||||
monkeypatch,
|
||||
{
|
||||
_V2_LIMIT_PATH: None,
|
||||
_V1_LIMIT_PATH: f"{8 * GiB_bytes}\n",
|
||||
_V1_USAGE_PATH: f"{2 * GiB_bytes}\n",
|
||||
},
|
||||
)
|
||||
|
||||
assert cru.get_cgroup_memory_limit() == 8 * GiB_bytes
|
||||
assert cru.get_cgroup_memory_usage() == 2 * GiB_bytes
|
||||
|
||||
|
||||
def test_cgroup_v1_unlimited_sentinel_is_ignored(monkeypatch):
|
||||
"""An unlimited cgroup v1 sentinel must not be treated as a real limit."""
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
_stub_files(
|
||||
monkeypatch,
|
||||
{_V2_LIMIT_PATH: None, _V1_LIMIT_PATH: f"{(1 << 63) - 1}\n"},
|
||||
)
|
||||
|
||||
assert cru.get_cgroup_memory_limit() is None
|
||||
assert cru.get_cgroup_memory_usage() is None
|
||||
|
||||
|
||||
def test_cgroup_no_limit_files_present(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
_stub_files(monkeypatch, {_V2_LIMIT_PATH: None, _V1_LIMIT_PATH: None})
|
||||
|
||||
assert cru.get_cgroup_memory_limit() is None
|
||||
assert cru.get_cgroup_memory_usage() is None
|
||||
|
||||
|
||||
def test_cgroup_skipped_on_non_linux(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "darwin")
|
||||
_stub_files(monkeypatch, {_V2_LIMIT_PATH: f"{1 * GiB_bytes}\n"})
|
||||
|
||||
assert cru.get_cgroup_memory_limit() is None
|
||||
assert cru.get_cgroup_memory_usage() is None
|
||||
|
||||
|
||||
def test_cgroup_limit_is_cached_until_explicitly_cleared(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
files = {
|
||||
_V2_LIMIT_PATH: f"{1 * GiB_bytes}\n",
|
||||
_V2_USAGE_PATH: f"{512 << 20}\n",
|
||||
}
|
||||
_stub_files(monkeypatch, files)
|
||||
|
||||
assert cru.get_cgroup_memory_limit() == 1 * GiB_bytes
|
||||
files[_V2_LIMIT_PATH] = f"{2 * GiB_bytes}\n"
|
||||
assert cru.get_cgroup_memory_limit() == 1 * GiB_bytes
|
||||
|
||||
cru.get_cgroup_memory_limit.cache_clear()
|
||||
assert cru.get_cgroup_memory_limit() == 2 * GiB_bytes
|
||||
|
||||
|
||||
def test_cgroup_usage_is_read_without_cache(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
files = {
|
||||
_V2_LIMIT_PATH: f"{1 * GiB_bytes}\n",
|
||||
_V2_USAGE_PATH: f"{512 << 20}\n",
|
||||
}
|
||||
_stub_files(monkeypatch, files)
|
||||
|
||||
assert cru.get_cgroup_memory_usage() == 512 << 20
|
||||
files[_V2_USAGE_PATH] = f"{768 << 20}\n"
|
||||
assert cru.get_cgroup_memory_usage() == 768 << 20
|
||||
|
||||
|
||||
def test_check_cgroup_memory_available_warns_on_low_headroom(monkeypatch):
|
||||
monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: 1 << 30)
|
||||
monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: 512 << 20)
|
||||
|
||||
with (
|
||||
mock.patch.object(cru.logger, "debug") as log_debug,
|
||||
mock.patch.object(cru.logger, "warning") as log_warning,
|
||||
):
|
||||
cru.check_cgroup_memory_available(600 << 20, "mmap")
|
||||
|
||||
log_debug.assert_not_called()
|
||||
log_warning.assert_called_once()
|
||||
assert "current headroom is below" in log_warning.call_args.args[-1]
|
||||
|
||||
|
||||
def test_check_cgroup_memory_available_logs_success_at_debug(monkeypatch):
|
||||
monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: 1 << 30)
|
||||
monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: 512 << 20)
|
||||
|
||||
with (
|
||||
mock.patch.object(cru.logger, "debug") as log_debug,
|
||||
mock.patch.object(cru.logger, "warning") as log_warning,
|
||||
):
|
||||
cru.check_cgroup_memory_available(256 << 20, "mmap")
|
||||
|
||||
log_debug.assert_called_once()
|
||||
log_warning.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("limit", "usage"),
|
||||
[(None, 512 << 20), (1 << 30, None)],
|
||||
)
|
||||
def test_check_cgroup_memory_available_skips_without_snapshot(
|
||||
monkeypatch, limit, usage
|
||||
):
|
||||
monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: limit)
|
||||
monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: usage)
|
||||
|
||||
with (
|
||||
mock.patch.object(cru.logger, "debug") as log_debug,
|
||||
mock.patch.object(cru.logger, "warning") as log_warning,
|
||||
):
|
||||
cru.check_cgroup_memory_available(1 << 60, "mmap")
|
||||
|
||||
log_debug.assert_not_called()
|
||||
log_warning.assert_not_called()
|
||||
@@ -906,12 +906,13 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
|
||||
monkeypatch.setattr(region.os, "open", mock_open)
|
||||
monkeypatch.setattr(region.os, "unlink", mock_unlink)
|
||||
monkeypatch.setattr(region.os, "close", mock_close)
|
||||
mock_check = MagicMock(
|
||||
side_effect=RuntimeError("Insufficient space in /dev/shm: 30 GB required.")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
region,
|
||||
"check_shm_free_space",
|
||||
lambda *a, **kw: (_ for _ in ()).throw(
|
||||
RuntimeError("Insufficient space in /dev/shm: 30 GB required.")
|
||||
),
|
||||
mock_check,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Insufficient space"):
|
||||
@@ -925,6 +926,10 @@ def test_insufficient_space_raises_clear_error(monkeypatch):
|
||||
|
||||
mock_unlink.assert_called_once_with(mmap_path)
|
||||
mock_close.assert_called_once_with(9999)
|
||||
mock_check.assert_called_once_with(
|
||||
4 * PAGE_SIZE,
|
||||
allocation_name="CPU KV offload shared region in /dev/shm",
|
||||
)
|
||||
|
||||
|
||||
def test_ftruncate_failure_cleans_up_creator(monkeypatch):
|
||||
|
||||
@@ -34,6 +34,7 @@ import vllm.envs as envs
|
||||
from vllm.distributed.utils import StatelessProcessGroup, sched_yield
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.cpu_resource_utils import check_cgroup_memory_available
|
||||
from vllm.utils.network_utils import (
|
||||
get_ip,
|
||||
get_open_zmq_inproc_path,
|
||||
@@ -224,26 +225,37 @@ class SpinCondition:
|
||||
SHM_PATH = "/dev/shm"
|
||||
|
||||
|
||||
def check_shm_free_space(required_bytes: int, shm_path: str = SHM_PATH) -> None:
|
||||
"""Raise if ``shm_path`` cannot fit a ``required_bytes`` shared segment.
|
||||
def check_shm_free_space(
|
||||
required_bytes: int,
|
||||
shm_path: str = SHM_PATH,
|
||||
*,
|
||||
allocation_name: str = "shared-memory allocation",
|
||||
) -> None:
|
||||
"""Raise if SHM cannot fit a shared segment and log cgroup headroom.
|
||||
|
||||
Args:
|
||||
required_bytes: Size of the shared-memory segment to be created.
|
||||
shm_path: Mount point backing POSIX shared memory; skipped if absent.
|
||||
shm_path: Mount point backing POSIX shared memory; its filesystem
|
||||
check is skipped if absent.
|
||||
allocation_name: Human-readable name used in errors and logs.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If ``required_bytes`` exceeds the free space.
|
||||
RuntimeError: If the SHM filesystem has insufficient space.
|
||||
"""
|
||||
if not os.path.isdir(shm_path):
|
||||
return
|
||||
free_bytes = shutil.disk_usage(shm_path).free
|
||||
if required_bytes <= free_bytes:
|
||||
return
|
||||
mib = 1 << 20
|
||||
raise RuntimeError(
|
||||
f"Insufficient space in {shm_path}: {required_bytes / mib:.0f} MiB "
|
||||
f"required, {free_bytes / mib:.0f} MiB free. Increase {shm_path} "
|
||||
"(e.g. --shm-size or --ipc=host)."
|
||||
if os.path.isdir(shm_path):
|
||||
free_bytes = shutil.disk_usage(shm_path).free
|
||||
if required_bytes > free_bytes:
|
||||
mib = 1 << 20
|
||||
raise RuntimeError(
|
||||
f"Insufficient space in {shm_path} for {allocation_name}: "
|
||||
f"{required_bytes / mib:.0f} MiB required, "
|
||||
f"{free_bytes / mib:.0f} MiB free. Increase {shm_path} "
|
||||
"(e.g. --shm-size or --ipc=host)."
|
||||
)
|
||||
|
||||
check_cgroup_memory_available(
|
||||
required_bytes,
|
||||
allocation_name,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -727,11 +727,15 @@ def _get_available_ram_bytes() -> int:
|
||||
|
||||
host_available = psutil.virtual_memory().available
|
||||
|
||||
from vllm.utils.cpu_resource_utils import get_cgroup_memory_limit
|
||||
from vllm.utils.cpu_resource_utils import (
|
||||
get_cgroup_memory_limit,
|
||||
get_cgroup_memory_usage,
|
||||
)
|
||||
|
||||
cgroup_limit, cgroup_usage = get_cgroup_memory_limit()
|
||||
cgroup_limit = get_cgroup_memory_limit()
|
||||
if cgroup_limit is None:
|
||||
return host_available
|
||||
cgroup_usage = get_cgroup_memory_usage()
|
||||
cgroup_available = (
|
||||
cgroup_limit if cgroup_usage is None else max(0, cgroup_limit - cgroup_usage)
|
||||
)
|
||||
|
||||
@@ -12,8 +12,12 @@ from functools import cache
|
||||
import psutil
|
||||
import regex as re
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
DEVICE_CONTROL_ENV_VAR = "CPU_VISIBLE_MEMORY_NODES"
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LogicalCPUInfo:
|
||||
@@ -63,21 +67,20 @@ def _read_int_file(path: str) -> int | None:
|
||||
|
||||
|
||||
@cache
|
||||
def get_cgroup_memory_limit() -> tuple[int | None, int | None]:
|
||||
"""Return (limit, usage) in bytes from cgroup, or (None, None).
|
||||
def get_cgroup_memory_limit() -> int | None:
|
||||
"""Return the cgroup memory limit in bytes, or None.
|
||||
|
||||
Supports both cgroup v2 (unified) and v1. Returns (None, None) when
|
||||
Supports both cgroup v2 (unified) and v1. Returns None when
|
||||
not running under a constrained cgroup (e.g. bare metal, or limit
|
||||
reported as `max`/an unrealistically large value).
|
||||
"""
|
||||
if sys.platform != "linux":
|
||||
return None, None
|
||||
return None
|
||||
|
||||
# cgroup v2 unified hierarchy
|
||||
v2_limit = _read_int_file("/sys/fs/cgroup/memory.max")
|
||||
if v2_limit is not None:
|
||||
v2_usage = _read_int_file("/sys/fs/cgroup/memory.current")
|
||||
return v2_limit, v2_usage
|
||||
return v2_limit
|
||||
|
||||
# cgroup v1
|
||||
v1_limit = _read_int_file("/sys/fs/cgroup/memory/memory.limit_in_bytes")
|
||||
@@ -85,11 +88,74 @@ def get_cgroup_memory_limit() -> tuple[int | None, int | None]:
|
||||
# cgroup v1 reports a huge sentinel (close to PAGE_COUNTER_MAX)
|
||||
# when unlimited. Treat absurdly large values as "no limit".
|
||||
if v1_limit >= (1 << 62):
|
||||
return None, None
|
||||
v1_usage = _read_int_file("/sys/fs/cgroup/memory/memory.usage_in_bytes")
|
||||
return v1_limit, v1_usage
|
||||
return None
|
||||
return v1_limit
|
||||
|
||||
return None, None
|
||||
return None
|
||||
|
||||
|
||||
def get_cgroup_memory_usage() -> int | None:
|
||||
"""Return the current cgroup memory usage in bytes, or None.
|
||||
|
||||
The usage value is intentionally read on every call because cgroup
|
||||
memory usage changes while the process is running.
|
||||
"""
|
||||
if sys.platform != "linux":
|
||||
return None
|
||||
|
||||
# cgroup v2 unified hierarchy
|
||||
if _read_int_file("/sys/fs/cgroup/memory.max") is not None:
|
||||
return _read_int_file("/sys/fs/cgroup/memory.current")
|
||||
|
||||
# cgroup v1
|
||||
v1_limit = _read_int_file("/sys/fs/cgroup/memory/memory.limit_in_bytes")
|
||||
if v1_limit is not None and v1_limit < (1 << 62):
|
||||
return _read_int_file("/sys/fs/cgroup/memory/memory.usage_in_bytes")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def check_cgroup_memory_available(
|
||||
required_bytes: int,
|
||||
allocation_name: str,
|
||||
) -> None:
|
||||
"""Log cgroup memory headroom for an upcoming allocation.
|
||||
|
||||
Args:
|
||||
required_bytes: Bytes required by the allocation.
|
||||
allocation_name: Human-readable name used in log messages.
|
||||
|
||||
Low headroom logs a warning, but does not reject the allocation because
|
||||
cgroup usage can include reclaimable memory. If the cgroup limit or usage
|
||||
cannot be read, the check is skipped.
|
||||
"""
|
||||
cgroup_limit = get_cgroup_memory_limit()
|
||||
cgroup_usage = get_cgroup_memory_usage()
|
||||
if cgroup_limit is None or cgroup_usage is None:
|
||||
return
|
||||
|
||||
cgroup_available = max(0, cgroup_limit - cgroup_usage)
|
||||
mib = 1 << 20
|
||||
remaining_bytes = cgroup_available - required_bytes
|
||||
log_fn = logger.debug if remaining_bytes >= 0 else logger.warning
|
||||
status = (
|
||||
"current headroom meets the requested allocation"
|
||||
if remaining_bytes >= 0
|
||||
else "current headroom is below the requested allocation; allocation "
|
||||
"will still be attempted because cgroup usage may be reclaimable"
|
||||
)
|
||||
log_fn(
|
||||
"Cgroup memory preflight for %s: %.0f MiB required, %.0f MiB current "
|
||||
"usage, %.0f MiB available under %.0f MiB limit, %.0f MiB remaining "
|
||||
"after allocation based on current usage; %s.",
|
||||
allocation_name,
|
||||
required_bytes / mib,
|
||||
cgroup_usage / mib,
|
||||
cgroup_available / mib,
|
||||
cgroup_limit / mib,
|
||||
remaining_bytes / mib,
|
||||
status,
|
||||
)
|
||||
|
||||
|
||||
def get_memory_affinity(pid: int = 0) -> list[int]:
|
||||
@@ -161,7 +227,8 @@ def get_memory_node_info(node_id: int = 0) -> MemoryNodeInfo:
|
||||
# would be applied to host RAM instead of the pod's limit. cgroup
|
||||
# does not expose per-NUMA-node limits, so we just clamp the totals
|
||||
# against the pod-wide limit here.
|
||||
cgroup_limit, cgroup_usage = get_cgroup_memory_limit()
|
||||
cgroup_limit = get_cgroup_memory_limit()
|
||||
cgroup_usage = get_cgroup_memory_usage()
|
||||
if cgroup_limit is not None and cgroup_limit < total_memory:
|
||||
total_memory = cgroup_limit
|
||||
cgroup_available = cgroup_limit - (cgroup_usage or 0)
|
||||
|
||||
@@ -135,7 +135,10 @@ class SharedOffloadRegion:
|
||||
self._creator = True
|
||||
if creator_memory_check is not None:
|
||||
creator_memory_check(self.total_size_bytes)
|
||||
check_shm_free_space(self.total_size_bytes)
|
||||
check_shm_free_space(
|
||||
self.total_size_bytes,
|
||||
allocation_name="CPU KV offload shared region in /dev/shm",
|
||||
)
|
||||
os.ftruncate(self.fd, self.total_size_bytes)
|
||||
logger.info(
|
||||
"Created mmap file %s (%.2f GB)",
|
||||
|
||||
Reference in New Issue
Block a user