From 9833aa53d5a9559b4cc8d174ae49d6e282c30423 Mon Sep 17 00:00:00 2001 From: AlexHuang Date: Thu, 6 Aug 2026 01:59:55 +0800 Subject: [PATCH] [Bugfix] Fail fast with a clear error when CPU offload region exceeds available space (#50358) Signed-off-by: Alex Co-authored-by: OpenAI Codex Co-authored-by: Or Ozeri --- .../cpu/test_shared_offload_region.py | 70 +++++++++++++++++++ .../kv_offload/cpu/shared_offload_region.py | 31 ++++++-- 2 files changed, 95 insertions(+), 6 deletions(-) diff --git a/tests/v1/kv_offload/cpu/test_shared_offload_region.py b/tests/v1/kv_offload/cpu/test_shared_offload_region.py index ca7295f197..dd6debfd43 100644 --- a/tests/v1/kv_offload/cpu/test_shared_offload_region.py +++ b/tests/v1/kv_offload/cpu/test_shared_offload_region.py @@ -8,6 +8,7 @@ import os import threading import time import uuid +from unittest.mock import MagicMock import pytest @@ -610,3 +611,72 @@ def test_wait_for_file_size_timeout(tmp_path): _wait_for_file_size(fd, PAGE_SIZE, timeout=0.1) finally: os.close(fd) + + +# --------------------------------------------------------------------------- +# Constructor — capacity validation +# --------------------------------------------------------------------------- + + +def test_insufficient_space_raises_clear_error(monkeypatch): + """A failed creator capacity check must clean up and give a clear error.""" + import vllm.v1.kv_offload.cpu.shared_offload_region as region + + engine_id = str(uuid.uuid4()) + mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap" + mock_open = MagicMock(return_value=9999) + mock_unlink = MagicMock() + mock_close = MagicMock() + monkeypatch.setattr(region.os, "open", mock_open) + monkeypatch.setattr(region.os, "unlink", mock_unlink) + monkeypatch.setattr(region.os, "close", mock_close) + monkeypatch.setattr( + region, + "check_shm_free_space", + lambda *a, **kw: (_ for _ in ()).throw( + RuntimeError("Insufficient space in /dev/shm: 30 GB required.") + ), + ) + + with pytest.raises(RuntimeError, match="Insufficient space"): + SharedOffloadRegion( + engine_id=engine_id, + num_blocks=4, + rank=0, + kv_bytes_per_block=PAGE_SIZE, + cpu_page_size=PAGE_SIZE, + ) + + mock_unlink.assert_called_once_with(mmap_path) + mock_close.assert_called_once_with(9999) + + +def test_ftruncate_failure_cleans_up_creator(monkeypatch): + """A failed creator ftruncate must close and unlink before re-raising.""" + import vllm.v1.kv_offload.cpu.shared_offload_region as region + + engine_id = str(uuid.uuid4()) + mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap" + mock_unlink = MagicMock() + mock_close = MagicMock() + monkeypatch.setattr(region.os, "open", MagicMock(return_value=9999)) + monkeypatch.setattr(region.os, "unlink", mock_unlink) + monkeypatch.setattr(region.os, "close", mock_close) + monkeypatch.setattr(region, "check_shm_free_space", MagicMock()) + monkeypatch.setattr( + region.os, + "ftruncate", + MagicMock(side_effect=OSError("ftruncate failed")), + ) + + with pytest.raises(OSError, match="ftruncate failed"): + SharedOffloadRegion( + engine_id=engine_id, + num_blocks=4, + rank=0, + kv_bytes_per_block=PAGE_SIZE, + cpu_page_size=PAGE_SIZE, + ) + + mock_unlink.assert_called_once_with(mmap_path) + mock_close.assert_called_once_with(9999) diff --git a/vllm/v1/kv_offload/cpu/shared_offload_region.py b/vllm/v1/kv_offload/cpu/shared_offload_region.py index 290c0d5107..8456f37c71 100644 --- a/vllm/v1/kv_offload/cpu/shared_offload_region.py +++ b/vllm/v1/kv_offload/cpu/shared_offload_region.py @@ -6,6 +6,9 @@ import time import torch +from vllm.distributed.device_communicators.shm_broadcast import ( + check_shm_free_space, +) from vllm.logger import init_logger from vllm.platforms import current_platform @@ -62,21 +65,37 @@ class SharedOffloadRegion: # exclusive upper bound for this worker's area within each row self._worker_area_end = (rank + 1) * cpu_page_size try: - # Exclusive create — only one worker succeeds self.fd: int | None = os.open( self.mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600 ) - os.ftruncate(self.fd, self.total_size_bytes) + except FileExistsError: + # Joiner path — another worker won O_EXCL. Reopen and wait + # for the file to reach expected size. + self.fd = os.open(self.mmap_path, os.O_RDWR) + try: + _wait_for_file_size(self.fd, self.total_size_bytes) + except (TimeoutError, OSError): + os.close(self.fd) + raise + logger.info("Opened existing mmap file %s", self.mmap_path) + else: + # Creator path. We won O_EXCL, so we own the file: any + # failure here must clean up so concurrent joiners don't + # land on a 0-byte stub and spin in _wait_for_file_size + # for the full 30 s timeout. + try: + check_shm_free_space(self.total_size_bytes) + os.ftruncate(self.fd, self.total_size_bytes) + except (RuntimeError, OSError): + os.unlink(self.mmap_path) + os.close(self.fd) + raise self._creator = True logger.info( "Created mmap file %s (%.2f GB)", self.mmap_path, self.total_size_bytes / 1e9, ) - except FileExistsError: - self.fd = os.open(self.mmap_path, os.O_RDWR) - _wait_for_file_size(self.fd, self.total_size_bytes) - logger.info("Opened existing mmap file %s", self.mmap_path) self.mmap_obj: mmap.mmap | None = mmap.mmap( self.fd,