[Bugfix] Fail fast with a clear error when CPU offload region exceeds available space (#50358)

Signed-off-by: Alex <jihui.huang@daocloud.io>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
AlexHuang
2026-08-05 20:59:55 +03:00
committed by GitHub
co-authored by OpenAI Codex Or Ozeri
parent e8586b2878
commit 9833aa53d5
2 changed files with 95 additions and 6 deletions
@@ -8,6 +8,7 @@ import os
import threading
import time
import uuid
from unittest.mock import MagicMock
import pytest
@@ -610,3 +611,72 @@ def test_wait_for_file_size_timeout(tmp_path):
_wait_for_file_size(fd, PAGE_SIZE, timeout=0.1)
finally:
os.close(fd)
# ---------------------------------------------------------------------------
# Constructor — capacity validation
# ---------------------------------------------------------------------------
def test_insufficient_space_raises_clear_error(monkeypatch):
"""A failed creator capacity check must clean up and give a clear error."""
import vllm.v1.kv_offload.cpu.shared_offload_region as region
engine_id = str(uuid.uuid4())
mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
mock_open = MagicMock(return_value=9999)
mock_unlink = MagicMock()
mock_close = MagicMock()
monkeypatch.setattr(region.os, "open", mock_open)
monkeypatch.setattr(region.os, "unlink", mock_unlink)
monkeypatch.setattr(region.os, "close", mock_close)
monkeypatch.setattr(
region,
"check_shm_free_space",
lambda *a, **kw: (_ for _ in ()).throw(
RuntimeError("Insufficient space in /dev/shm: 30 GB required.")
),
)
with pytest.raises(RuntimeError, match="Insufficient space"):
SharedOffloadRegion(
engine_id=engine_id,
num_blocks=4,
rank=0,
kv_bytes_per_block=PAGE_SIZE,
cpu_page_size=PAGE_SIZE,
)
mock_unlink.assert_called_once_with(mmap_path)
mock_close.assert_called_once_with(9999)
def test_ftruncate_failure_cleans_up_creator(monkeypatch):
"""A failed creator ftruncate must close and unlink before re-raising."""
import vllm.v1.kv_offload.cpu.shared_offload_region as region
engine_id = str(uuid.uuid4())
mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
mock_unlink = MagicMock()
mock_close = MagicMock()
monkeypatch.setattr(region.os, "open", MagicMock(return_value=9999))
monkeypatch.setattr(region.os, "unlink", mock_unlink)
monkeypatch.setattr(region.os, "close", mock_close)
monkeypatch.setattr(region, "check_shm_free_space", MagicMock())
monkeypatch.setattr(
region.os,
"ftruncate",
MagicMock(side_effect=OSError("ftruncate failed")),
)
with pytest.raises(OSError, match="ftruncate failed"):
SharedOffloadRegion(
engine_id=engine_id,
num_blocks=4,
rank=0,
kv_bytes_per_block=PAGE_SIZE,
cpu_page_size=PAGE_SIZE,
)
mock_unlink.assert_called_once_with(mmap_path)
mock_close.assert_called_once_with(9999)
@@ -6,6 +6,9 @@ import time
import torch
from vllm.distributed.device_communicators.shm_broadcast import (
check_shm_free_space,
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
@@ -62,21 +65,37 @@ class SharedOffloadRegion:
# exclusive upper bound for this worker's area within each row
self._worker_area_end = (rank + 1) * cpu_page_size
try:
# Exclusive create — only one worker succeeds
self.fd: int | None = os.open(
self.mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600
)
os.ftruncate(self.fd, self.total_size_bytes)
except FileExistsError:
# Joiner path — another worker won O_EXCL. Reopen and wait
# for the file to reach expected size.
self.fd = os.open(self.mmap_path, os.O_RDWR)
try:
_wait_for_file_size(self.fd, self.total_size_bytes)
except (TimeoutError, OSError):
os.close(self.fd)
raise
logger.info("Opened existing mmap file %s", self.mmap_path)
else:
# Creator path. We won O_EXCL, so we own the file: any
# failure here must clean up so concurrent joiners don't
# land on a 0-byte stub and spin in _wait_for_file_size
# for the full 30 s timeout.
try:
check_shm_free_space(self.total_size_bytes)
os.ftruncate(self.fd, self.total_size_bytes)
except (RuntimeError, OSError):
os.unlink(self.mmap_path)
os.close(self.fd)
raise
self._creator = True
logger.info(
"Created mmap file %s (%.2f GB)",
self.mmap_path,
self.total_size_bytes / 1e9,
)
except FileExistsError:
self.fd = os.open(self.mmap_path, os.O_RDWR)
_wait_for_file_size(self.fd, self.total_size_bytes)
logger.info("Opened existing mmap file %s", self.mmap_path)
self.mmap_obj: mmap.mmap | None = mmap.mmap(
self.fd,