[Core] Fail fast when /dev/shm is too small for the shm ring buffer (#48879)
Signed-off-by: Dr Andrea Tassi <andrea@verticular.uk> Co-authored-by: Dr Andrea Tassi <andrea@verticular.uk> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Dr Andrea Tassi
Claude Opus 4.8
parent
fd10e8946d
commit
831d3848f1
@@ -4,6 +4,7 @@
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import multiprocess as mp
|
||||
@@ -11,7 +12,12 @@ import numpy as np
|
||||
import pytest
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
|
||||
from vllm.distributed.device_communicators import shm_broadcast
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
MessageQueue,
|
||||
ShmRingBuffer,
|
||||
check_shm_free_space,
|
||||
)
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
@@ -522,3 +528,39 @@ def test_warning_logs(caplog_vllm):
|
||||
# Clean up when done
|
||||
writer.shutdown()
|
||||
reader.shutdown()
|
||||
|
||||
|
||||
def _fake_disk_usage(free_bytes: int):
|
||||
return SimpleNamespace(total=free_bytes, used=0, free=free_bytes)
|
||||
|
||||
|
||||
def test_check_shm_free_space_raises_when_insufficient(tmp_path):
|
||||
with (
|
||||
mock.patch.object(
|
||||
shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(32 << 20)
|
||||
),
|
||||
pytest.raises(RuntimeError, match="Insufficient space"),
|
||||
):
|
||||
check_shm_free_space(240 << 20, shm_path=str(tmp_path))
|
||||
|
||||
|
||||
def test_check_shm_free_space_passes_when_sufficient(tmp_path):
|
||||
with mock.patch.object(
|
||||
shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(512 << 20)
|
||||
):
|
||||
check_shm_free_space(240 << 20, shm_path=str(tmp_path))
|
||||
|
||||
|
||||
def test_check_shm_free_space_skipped_when_path_missing(tmp_path):
|
||||
check_shm_free_space(1 << 60, shm_path=str(tmp_path / "does-not-exist"))
|
||||
|
||||
|
||||
def test_shm_ring_buffer_creation_checks_free_space():
|
||||
with (
|
||||
mock.patch.object(
|
||||
shm_broadcast.shutil, "disk_usage", return_value=_fake_disk_usage(1 << 20)
|
||||
),
|
||||
mock.patch.object(shm_broadcast.os.path, "isdir", return_value=True),
|
||||
pytest.raises(RuntimeError, match="Insufficient space"),
|
||||
):
|
||||
ShmRingBuffer(n_reader=1, max_chunk_bytes=24 * 1024 * 1024, max_chunks=10)
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import functools
|
||||
import os
|
||||
import pickle
|
||||
import shutil
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
@@ -218,6 +220,32 @@ class SpinCondition:
|
||||
self.local_notify_socket.send(b"\x00")
|
||||
|
||||
|
||||
SHM_PATH = "/dev/shm"
|
||||
|
||||
|
||||
def check_shm_free_space(required_bytes: int, shm_path: str = SHM_PATH) -> None:
|
||||
"""Raise if ``shm_path`` cannot fit a ``required_bytes`` shared segment.
|
||||
|
||||
Args:
|
||||
required_bytes: Size of the shared-memory segment to be created.
|
||||
shm_path: Mount point backing POSIX shared memory; skipped if absent.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If ``required_bytes`` exceeds the free space.
|
||||
"""
|
||||
if not os.path.isdir(shm_path):
|
||||
return
|
||||
free_bytes = shutil.disk_usage(shm_path).free
|
||||
if required_bytes <= free_bytes:
|
||||
return
|
||||
mib = 1 << 20
|
||||
raise RuntimeError(
|
||||
f"Insufficient space in {shm_path}: {required_bytes / mib:.0f} MiB "
|
||||
f"required, {free_bytes / mib:.0f} MiB free. Increase {shm_path} "
|
||||
"(e.g. --shm-size or --ipc=host)."
|
||||
)
|
||||
|
||||
|
||||
class ShmRingBuffer:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -288,6 +316,7 @@ class ShmRingBuffer:
|
||||
if name is None:
|
||||
# we are creating a buffer
|
||||
self.is_creator = True
|
||||
check_shm_free_space(self.total_bytes_of_buffer)
|
||||
self.shared_memory = shared_memory.SharedMemory(
|
||||
create=True, size=self.total_bytes_of_buffer
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user