From 0f8fa53acf43a9a6a7a1f7e6b196950c78cd6d7a Mon Sep 17 00:00:00 2001 From: AlexHuang Date: Wed, 16 Sep 2026 14:17:50 +0800 Subject: [PATCH] [Bugfix][KV Offload] Check cgroup memory before SHM allocation (#54014) Signed-off-by: Alex Signed-off-by: AlexHuang Signed-off-by: Alex Co-authored-by: OpenAI Codex Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> --- tests/distributed/test_shm_broadcast.py | 35 +++- tests/utils_/test_cpu_resource_utils.py | 189 ++++++++++++++++++ .../cpu/test_shared_offload_region.py | 11 +- .../device_communicators/shm_broadcast.py | 40 ++-- .../model_loader/weight_utils.py | 8 +- vllm/utils/cpu_resource_utils.py | 89 ++++++++- .../kv_offload/cpu/shared_offload_region.py | 5 +- 7 files changed, 343 insertions(+), 34 deletions(-) create mode 100644 tests/utils_/test_cpu_resource_utils.py diff --git a/tests/distributed/test_shm_broadcast.py b/tests/distributed/test_shm_broadcast.py index c3833b6b9f..ef7f523b78 100644 --- a/tests/distributed/test_shm_broadcast.py +++ b/tests/distributed/test_shm_broadcast.py @@ -694,14 +694,43 @@ def test_check_shm_free_space_raises_when_insufficient(tmp_path): def test_check_shm_free_space_passes_when_sufficient(tmp_path): - with mock.patch.object( - shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(512 << 20) + with ( + mock.patch.object( + shm_broadcast.shutil, + "disk_usage", + return_value=_fake_disk_usage(512 << 20), + ), + mock.patch.object(shm_broadcast, "check_cgroup_memory_available"), ): check_shm_free_space(240 << 20, shm_path=str(tmp_path)) def test_check_shm_free_space_skipped_when_path_missing(tmp_path): - check_shm_free_space(1 << 60, shm_path=str(tmp_path / "does-not-exist")) + with mock.patch.object(shm_broadcast, "check_cgroup_memory_available"): + check_shm_free_space(1 << 60, shm_path=str(tmp_path / "does-not-exist")) + + +def test_check_shm_free_space_checks_cgroup(tmp_path): + with ( + mock.patch.object( + shm_broadcast.shutil, + "disk_usage", + return_value=_fake_disk_usage(512 << 20), + ), + mock.patch.object( + shm_broadcast, "check_cgroup_memory_available" + ) as check_cgroup, + ): + check_shm_free_space( + 240 << 20, + shm_path=str(tmp_path), + allocation_name="SHM mmap", + ) + + check_cgroup.assert_called_once_with( + 240 << 20, + "SHM mmap", + ) def test_shm_ring_buffer_creation_checks_free_space(): diff --git a/tests/utils_/test_cpu_resource_utils.py b/tests/utils_/test_cpu_resource_utils.py new file mode 100644 index 0000000000..ff314d2231 --- /dev/null +++ b/tests/utils_/test_cpu_resource_utils.py @@ -0,0 +1,189 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Cgroup memory readers and SHM allocation preflight checks.""" + +import sys +from io import StringIO +from unittest import mock + +import pytest + +from vllm.utils import cpu_resource_utils as cru +from vllm.utils.mem_constants import GiB_bytes + +_V2_LIMIT_PATH = "/sys/fs/cgroup/memory.max" +_V2_USAGE_PATH = "/sys/fs/cgroup/memory.current" +_V1_LIMIT_PATH = "/sys/fs/cgroup/memory/memory.limit_in_bytes" +_V1_USAGE_PATH = "/sys/fs/cgroup/memory/memory.usage_in_bytes" + + +@pytest.fixture(autouse=True) +def _clear_cgroup_cache(): + cru.get_cgroup_memory_limit.cache_clear() + yield + cru.get_cgroup_memory_limit.cache_clear() + + +def _stub_files(monkeypatch, files: dict): + """Stub ``open()`` for a fixed set of paths; ``None`` -> OSError.""" + real_open = open + + def fake_open(path, *args, **kwargs): + if path in files: + content = files[path] + if content is None: + raise OSError(f"no such file: {path}") + return StringIO(content) + return real_open(path, *args, **kwargs) + + monkeypatch.setattr("builtins.open", fake_open) + + +def test_cgroup_v2_limit_and_usage(monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + _stub_files( + monkeypatch, + { + _V2_LIMIT_PATH: f"{20 * GiB_bytes}\n", + _V2_USAGE_PATH: f"{5 * GiB_bytes}\n", + }, + ) + + assert cru.get_cgroup_memory_limit() == 20 * GiB_bytes + assert cru.get_cgroup_memory_usage() == 5 * GiB_bytes + + +def test_cgroup_v2_unlimited_falls_back_to_v1(monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + _stub_files( + monkeypatch, + { + _V2_LIMIT_PATH: "max\n", + _V1_LIMIT_PATH: f"{8 * GiB_bytes}\n", + _V1_USAGE_PATH: f"{1 * GiB_bytes}\n", + }, + ) + + assert cru.get_cgroup_memory_limit() == 8 * GiB_bytes + assert cru.get_cgroup_memory_usage() == 1 * GiB_bytes + + +def test_cgroup_v1_limit_and_usage(monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + _stub_files( + monkeypatch, + { + _V2_LIMIT_PATH: None, + _V1_LIMIT_PATH: f"{8 * GiB_bytes}\n", + _V1_USAGE_PATH: f"{2 * GiB_bytes}\n", + }, + ) + + assert cru.get_cgroup_memory_limit() == 8 * GiB_bytes + assert cru.get_cgroup_memory_usage() == 2 * GiB_bytes + + +def test_cgroup_v1_unlimited_sentinel_is_ignored(monkeypatch): + """An unlimited cgroup v1 sentinel must not be treated as a real limit.""" + monkeypatch.setattr(sys, "platform", "linux") + _stub_files( + monkeypatch, + {_V2_LIMIT_PATH: None, _V1_LIMIT_PATH: f"{(1 << 63) - 1}\n"}, + ) + + assert cru.get_cgroup_memory_limit() is None + assert cru.get_cgroup_memory_usage() is None + + +def test_cgroup_no_limit_files_present(monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + _stub_files(monkeypatch, {_V2_LIMIT_PATH: None, _V1_LIMIT_PATH: None}) + + assert cru.get_cgroup_memory_limit() is None + assert cru.get_cgroup_memory_usage() is None + + +def test_cgroup_skipped_on_non_linux(monkeypatch): + monkeypatch.setattr(sys, "platform", "darwin") + _stub_files(monkeypatch, {_V2_LIMIT_PATH: f"{1 * GiB_bytes}\n"}) + + assert cru.get_cgroup_memory_limit() is None + assert cru.get_cgroup_memory_usage() is None + + +def test_cgroup_limit_is_cached_until_explicitly_cleared(monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + files = { + _V2_LIMIT_PATH: f"{1 * GiB_bytes}\n", + _V2_USAGE_PATH: f"{512 << 20}\n", + } + _stub_files(monkeypatch, files) + + assert cru.get_cgroup_memory_limit() == 1 * GiB_bytes + files[_V2_LIMIT_PATH] = f"{2 * GiB_bytes}\n" + assert cru.get_cgroup_memory_limit() == 1 * GiB_bytes + + cru.get_cgroup_memory_limit.cache_clear() + assert cru.get_cgroup_memory_limit() == 2 * GiB_bytes + + +def test_cgroup_usage_is_read_without_cache(monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + files = { + _V2_LIMIT_PATH: f"{1 * GiB_bytes}\n", + _V2_USAGE_PATH: f"{512 << 20}\n", + } + _stub_files(monkeypatch, files) + + assert cru.get_cgroup_memory_usage() == 512 << 20 + files[_V2_USAGE_PATH] = f"{768 << 20}\n" + assert cru.get_cgroup_memory_usage() == 768 << 20 + + +def test_check_cgroup_memory_available_warns_on_low_headroom(monkeypatch): + monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: 1 << 30) + monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: 512 << 20) + + with ( + mock.patch.object(cru.logger, "debug") as log_debug, + mock.patch.object(cru.logger, "warning") as log_warning, + ): + cru.check_cgroup_memory_available(600 << 20, "mmap") + + log_debug.assert_not_called() + log_warning.assert_called_once() + assert "current headroom is below" in log_warning.call_args.args[-1] + + +def test_check_cgroup_memory_available_logs_success_at_debug(monkeypatch): + monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: 1 << 30) + monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: 512 << 20) + + with ( + mock.patch.object(cru.logger, "debug") as log_debug, + mock.patch.object(cru.logger, "warning") as log_warning, + ): + cru.check_cgroup_memory_available(256 << 20, "mmap") + + log_debug.assert_called_once() + log_warning.assert_not_called() + + +@pytest.mark.parametrize( + ("limit", "usage"), + [(None, 512 << 20), (1 << 30, None)], +) +def test_check_cgroup_memory_available_skips_without_snapshot( + monkeypatch, limit, usage +): + monkeypatch.setattr(cru, "get_cgroup_memory_limit", lambda: limit) + monkeypatch.setattr(cru, "get_cgroup_memory_usage", lambda: usage) + + with ( + mock.patch.object(cru.logger, "debug") as log_debug, + mock.patch.object(cru.logger, "warning") as log_warning, + ): + cru.check_cgroup_memory_available(1 << 60, "mmap") + + log_debug.assert_not_called() + log_warning.assert_not_called() diff --git a/tests/v1/kv_offload/cpu/test_shared_offload_region.py b/tests/v1/kv_offload/cpu/test_shared_offload_region.py index c448e76a9c..4f22d122db 100644 --- a/tests/v1/kv_offload/cpu/test_shared_offload_region.py +++ b/tests/v1/kv_offload/cpu/test_shared_offload_region.py @@ -906,12 +906,13 @@ def test_insufficient_space_raises_clear_error(monkeypatch): monkeypatch.setattr(region.os, "open", mock_open) monkeypatch.setattr(region.os, "unlink", mock_unlink) monkeypatch.setattr(region.os, "close", mock_close) + mock_check = MagicMock( + side_effect=RuntimeError("Insufficient space in /dev/shm: 30 GB required.") + ) monkeypatch.setattr( region, "check_shm_free_space", - lambda *a, **kw: (_ for _ in ()).throw( - RuntimeError("Insufficient space in /dev/shm: 30 GB required.") - ), + mock_check, ) with pytest.raises(RuntimeError, match="Insufficient space"): @@ -925,6 +926,10 @@ def test_insufficient_space_raises_clear_error(monkeypatch): mock_unlink.assert_called_once_with(mmap_path) mock_close.assert_called_once_with(9999) + mock_check.assert_called_once_with( + 4 * PAGE_SIZE, + allocation_name="CPU KV offload shared region in /dev/shm", + ) def test_ftruncate_failure_cleans_up_creator(monkeypatch): diff --git a/vllm/distributed/device_communicators/shm_broadcast.py b/vllm/distributed/device_communicators/shm_broadcast.py index 6caacea6ec..2171f79fb7 100644 --- a/vllm/distributed/device_communicators/shm_broadcast.py +++ b/vllm/distributed/device_communicators/shm_broadcast.py @@ -34,6 +34,7 @@ import vllm.envs as envs from vllm.distributed.utils import StatelessProcessGroup, sched_yield from vllm.logger import init_logger from vllm.platforms import current_platform +from vllm.utils.cpu_resource_utils import check_cgroup_memory_available from vllm.utils.network_utils import ( get_ip, get_open_zmq_inproc_path, @@ -224,26 +225,37 @@ class SpinCondition: SHM_PATH = "/dev/shm" -def check_shm_free_space(required_bytes: int, shm_path: str = SHM_PATH) -> None: - """Raise if ``shm_path`` cannot fit a ``required_bytes`` shared segment. +def check_shm_free_space( + required_bytes: int, + shm_path: str = SHM_PATH, + *, + allocation_name: str = "shared-memory allocation", +) -> None: + """Raise if SHM cannot fit a shared segment and log cgroup headroom. Args: required_bytes: Size of the shared-memory segment to be created. - shm_path: Mount point backing POSIX shared memory; skipped if absent. + shm_path: Mount point backing POSIX shared memory; its filesystem + check is skipped if absent. + allocation_name: Human-readable name used in errors and logs. Raises: - RuntimeError: If ``required_bytes`` exceeds the free space. + RuntimeError: If the SHM filesystem has insufficient space. """ - if not os.path.isdir(shm_path): - return - free_bytes = shutil.disk_usage(shm_path).free - if required_bytes <= free_bytes: - return - mib = 1 << 20 - raise RuntimeError( - f"Insufficient space in {shm_path}: {required_bytes / mib:.0f} MiB " - f"required, {free_bytes / mib:.0f} MiB free. Increase {shm_path} " - "(e.g. --shm-size or --ipc=host)." + if os.path.isdir(shm_path): + free_bytes = shutil.disk_usage(shm_path).free + if required_bytes > free_bytes: + mib = 1 << 20 + raise RuntimeError( + f"Insufficient space in {shm_path} for {allocation_name}: " + f"{required_bytes / mib:.0f} MiB required, " + f"{free_bytes / mib:.0f} MiB free. Increase {shm_path} " + "(e.g. --shm-size or --ipc=host)." + ) + + check_cgroup_memory_available( + required_bytes, + allocation_name, ) diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index abe24fa06e..d291f1929c 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -727,11 +727,15 @@ def _get_available_ram_bytes() -> int: host_available = psutil.virtual_memory().available - from vllm.utils.cpu_resource_utils import get_cgroup_memory_limit + from vllm.utils.cpu_resource_utils import ( + get_cgroup_memory_limit, + get_cgroup_memory_usage, + ) - cgroup_limit, cgroup_usage = get_cgroup_memory_limit() + cgroup_limit = get_cgroup_memory_limit() if cgroup_limit is None: return host_available + cgroup_usage = get_cgroup_memory_usage() cgroup_available = ( cgroup_limit if cgroup_usage is None else max(0, cgroup_limit - cgroup_usage) ) diff --git a/vllm/utils/cpu_resource_utils.py b/vllm/utils/cpu_resource_utils.py index 53e7461065..a1e7ab0ee3 100644 --- a/vllm/utils/cpu_resource_utils.py +++ b/vllm/utils/cpu_resource_utils.py @@ -12,8 +12,12 @@ from functools import cache import psutil import regex as re +from vllm.logger import init_logger + DEVICE_CONTROL_ENV_VAR = "CPU_VISIBLE_MEMORY_NODES" +logger = init_logger(__name__) + @dataclass class LogicalCPUInfo: @@ -63,21 +67,20 @@ def _read_int_file(path: str) -> int | None: @cache -def get_cgroup_memory_limit() -> tuple[int | None, int | None]: - """Return (limit, usage) in bytes from cgroup, or (None, None). +def get_cgroup_memory_limit() -> int | None: + """Return the cgroup memory limit in bytes, or None. - Supports both cgroup v2 (unified) and v1. Returns (None, None) when + Supports both cgroup v2 (unified) and v1. Returns None when not running under a constrained cgroup (e.g. bare metal, or limit reported as `max`/an unrealistically large value). """ if sys.platform != "linux": - return None, None + return None # cgroup v2 unified hierarchy v2_limit = _read_int_file("/sys/fs/cgroup/memory.max") if v2_limit is not None: - v2_usage = _read_int_file("/sys/fs/cgroup/memory.current") - return v2_limit, v2_usage + return v2_limit # cgroup v1 v1_limit = _read_int_file("/sys/fs/cgroup/memory/memory.limit_in_bytes") @@ -85,11 +88,74 @@ def get_cgroup_memory_limit() -> tuple[int | None, int | None]: # cgroup v1 reports a huge sentinel (close to PAGE_COUNTER_MAX) # when unlimited. Treat absurdly large values as "no limit". if v1_limit >= (1 << 62): - return None, None - v1_usage = _read_int_file("/sys/fs/cgroup/memory/memory.usage_in_bytes") - return v1_limit, v1_usage + return None + return v1_limit - return None, None + return None + + +def get_cgroup_memory_usage() -> int | None: + """Return the current cgroup memory usage in bytes, or None. + + The usage value is intentionally read on every call because cgroup + memory usage changes while the process is running. + """ + if sys.platform != "linux": + return None + + # cgroup v2 unified hierarchy + if _read_int_file("/sys/fs/cgroup/memory.max") is not None: + return _read_int_file("/sys/fs/cgroup/memory.current") + + # cgroup v1 + v1_limit = _read_int_file("/sys/fs/cgroup/memory/memory.limit_in_bytes") + if v1_limit is not None and v1_limit < (1 << 62): + return _read_int_file("/sys/fs/cgroup/memory/memory.usage_in_bytes") + + return None + + +def check_cgroup_memory_available( + required_bytes: int, + allocation_name: str, +) -> None: + """Log cgroup memory headroom for an upcoming allocation. + + Args: + required_bytes: Bytes required by the allocation. + allocation_name: Human-readable name used in log messages. + + Low headroom logs a warning, but does not reject the allocation because + cgroup usage can include reclaimable memory. If the cgroup limit or usage + cannot be read, the check is skipped. + """ + cgroup_limit = get_cgroup_memory_limit() + cgroup_usage = get_cgroup_memory_usage() + if cgroup_limit is None or cgroup_usage is None: + return + + cgroup_available = max(0, cgroup_limit - cgroup_usage) + mib = 1 << 20 + remaining_bytes = cgroup_available - required_bytes + log_fn = logger.debug if remaining_bytes >= 0 else logger.warning + status = ( + "current headroom meets the requested allocation" + if remaining_bytes >= 0 + else "current headroom is below the requested allocation; allocation " + "will still be attempted because cgroup usage may be reclaimable" + ) + log_fn( + "Cgroup memory preflight for %s: %.0f MiB required, %.0f MiB current " + "usage, %.0f MiB available under %.0f MiB limit, %.0f MiB remaining " + "after allocation based on current usage; %s.", + allocation_name, + required_bytes / mib, + cgroup_usage / mib, + cgroup_available / mib, + cgroup_limit / mib, + remaining_bytes / mib, + status, + ) def get_memory_affinity(pid: int = 0) -> list[int]: @@ -161,7 +227,8 @@ def get_memory_node_info(node_id: int = 0) -> MemoryNodeInfo: # would be applied to host RAM instead of the pod's limit. cgroup # does not expose per-NUMA-node limits, so we just clamp the totals # against the pod-wide limit here. - cgroup_limit, cgroup_usage = get_cgroup_memory_limit() + cgroup_limit = get_cgroup_memory_limit() + cgroup_usage = get_cgroup_memory_usage() if cgroup_limit is not None and cgroup_limit < total_memory: total_memory = cgroup_limit cgroup_available = cgroup_limit - (cgroup_usage or 0) diff --git a/vllm/v1/kv_offload/cpu/shared_offload_region.py b/vllm/v1/kv_offload/cpu/shared_offload_region.py index 46a1e7ae64..284f1de68e 100644 --- a/vllm/v1/kv_offload/cpu/shared_offload_region.py +++ b/vllm/v1/kv_offload/cpu/shared_offload_region.py @@ -135,7 +135,10 @@ class SharedOffloadRegion: self._creator = True if creator_memory_check is not None: creator_memory_check(self.total_size_bytes) - check_shm_free_space(self.total_size_bytes) + check_shm_free_space( + self.total_size_bytes, + allocation_name="CPU KV offload shared region in /dev/shm", + ) os.ftruncate(self.fd, self.total_size_bytes) logger.info( "Created mmap file %s (%.2f GB)",