From 831d3848f16e21e652bf1c377d010c2b5fe76f37 Mon Sep 17 00:00:00 2001 From: Andrea Tassi Date: Mon, 27 Jul 2026 20:48:35 +0100 Subject: [PATCH] [Core] Fail fast when /dev/shm is too small for the shm ring buffer (#48879) Signed-off-by: Dr Andrea Tassi Co-authored-by: Dr Andrea Tassi Co-authored-by: Claude Opus 4.8 --- tests/distributed/test_shm_broadcast.py | 44 ++++++++++++++++++- .../device_communicators/shm_broadcast.py | 29 ++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/tests/distributed/test_shm_broadcast.py b/tests/distributed/test_shm_broadcast.py index 33affb2a39..1795792405 100644 --- a/tests/distributed/test_shm_broadcast.py +++ b/tests/distributed/test_shm_broadcast.py @@ -4,6 +4,7 @@ import random import threading import time +from types import SimpleNamespace from unittest import mock import multiprocess as mp @@ -11,7 +12,12 @@ import numpy as np import pytest import torch.distributed as dist -from vllm.distributed.device_communicators.shm_broadcast import MessageQueue +from vllm.distributed.device_communicators import shm_broadcast +from vllm.distributed.device_communicators.shm_broadcast import ( + MessageQueue, + ShmRingBuffer, + check_shm_free_space, +) from vllm.distributed.utils import StatelessProcessGroup from vllm.utils.network_utils import get_open_port from vllm.utils.system_utils import update_environment_variables @@ -522,3 +528,39 @@ def test_warning_logs(caplog_vllm): # Clean up when done writer.shutdown() reader.shutdown() + + +def _fake_disk_usage(free_bytes: int): + return SimpleNamespace(total=free_bytes, used=0, free=free_bytes) + + +def test_check_shm_free_space_raises_when_insufficient(tmp_path): + with ( + mock.patch.object( + shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(32 << 20) + ), + pytest.raises(RuntimeError, match="Insufficient space"), + ): + check_shm_free_space(240 << 20, shm_path=str(tmp_path)) + + +def test_check_shm_free_space_passes_when_sufficient(tmp_path): + with mock.patch.object( + shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(512 << 20) + ): + check_shm_free_space(240 << 20, shm_path=str(tmp_path)) + + +def test_check_shm_free_space_skipped_when_path_missing(tmp_path): + check_shm_free_space(1 << 60, shm_path=str(tmp_path / "does-not-exist")) + + +def test_shm_ring_buffer_creation_checks_free_space(): + with ( + mock.patch.object( + shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(1 << 20) + ), + mock.patch.object(shm_broadcast.os.path, "isdir", return_value=True), + pytest.raises(RuntimeError, match="Insufficient space"), + ): + ShmRingBuffer(n_reader=1, max_chunk_bytes=24 * 1024 * 1024, max_chunks=10) diff --git a/vllm/distributed/device_communicators/shm_broadcast.py b/vllm/distributed/device_communicators/shm_broadcast.py index 6b9dd4068b..afabdf18c8 100644 --- a/vllm/distributed/device_communicators/shm_broadcast.py +++ b/vllm/distributed/device_communicators/shm_broadcast.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import functools +import os import pickle +import shutil import sys import threading import time @@ -218,6 +220,32 @@ class SpinCondition: self.local_notify_socket.send(b"\x00") +SHM_PATH = "/dev/shm" + + +def check_shm_free_space(required_bytes: int, shm_path: str = SHM_PATH) -> None: + """Raise if ``shm_path`` cannot fit a ``required_bytes`` shared segment. + + Args: + required_bytes: Size of the shared-memory segment to be created. + shm_path: Mount point backing POSIX shared memory; skipped if absent. + + Raises: + RuntimeError: If ``required_bytes`` exceeds the free space. + """ + if not os.path.isdir(shm_path): + return + free_bytes = shutil.disk_usage(shm_path).free + if required_bytes <= free_bytes: + return + mib = 1 << 20 + raise RuntimeError( + f"Insufficient space in {shm_path}: {required_bytes / mib:.0f} MiB " + f"required, {free_bytes / mib:.0f} MiB free. Increase {shm_path} " + "(e.g. --shm-size or --ipc=host)." + ) + + class ShmRingBuffer: def __init__( self, @@ -288,6 +316,7 @@ class ShmRingBuffer: if name is None: # we are creating a buffer self.is_creator = True + check_shm_free_space(self.total_bytes_of_buffer) self.shared_memory = shared_memory.SharedMemory( create=True, size=self.total_bytes_of_buffer )