[Bugfix] Fail fast with a clear error when CPU offload region exceeds available space (#50358)
Signed-off-by: Alex <jihui.huang@daocloud.io> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
OpenAI Codex
Or Ozeri
parent
e8586b2878
commit
9833aa53d5
@@ -8,6 +8,7 @@ import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -610,3 +611,72 @@ def test_wait_for_file_size_timeout(tmp_path):
|
||||
_wait_for_file_size(fd, PAGE_SIZE, timeout=0.1)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constructor — capacity validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_insufficient_space_raises_clear_error(monkeypatch):
|
||||
"""A failed creator capacity check must clean up and give a clear error."""
|
||||
import vllm.v1.kv_offload.cpu.shared_offload_region as region
|
||||
|
||||
engine_id = str(uuid.uuid4())
|
||||
mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
|
||||
mock_open = MagicMock(return_value=9999)
|
||||
mock_unlink = MagicMock()
|
||||
mock_close = MagicMock()
|
||||
monkeypatch.setattr(region.os, "open", mock_open)
|
||||
monkeypatch.setattr(region.os, "unlink", mock_unlink)
|
||||
monkeypatch.setattr(region.os, "close", mock_close)
|
||||
monkeypatch.setattr(
|
||||
region,
|
||||
"check_shm_free_space",
|
||||
lambda *a, **kw: (_ for _ in ()).throw(
|
||||
RuntimeError("Insufficient space in /dev/shm: 30 GB required.")
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Insufficient space"):
|
||||
SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=4,
|
||||
rank=0,
|
||||
kv_bytes_per_block=PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
)
|
||||
|
||||
mock_unlink.assert_called_once_with(mmap_path)
|
||||
mock_close.assert_called_once_with(9999)
|
||||
|
||||
|
||||
def test_ftruncate_failure_cleans_up_creator(monkeypatch):
|
||||
"""A failed creator ftruncate must close and unlink before re-raising."""
|
||||
import vllm.v1.kv_offload.cpu.shared_offload_region as region
|
||||
|
||||
engine_id = str(uuid.uuid4())
|
||||
mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
|
||||
mock_unlink = MagicMock()
|
||||
mock_close = MagicMock()
|
||||
monkeypatch.setattr(region.os, "open", MagicMock(return_value=9999))
|
||||
monkeypatch.setattr(region.os, "unlink", mock_unlink)
|
||||
monkeypatch.setattr(region.os, "close", mock_close)
|
||||
monkeypatch.setattr(region, "check_shm_free_space", MagicMock())
|
||||
monkeypatch.setattr(
|
||||
region.os,
|
||||
"ftruncate",
|
||||
MagicMock(side_effect=OSError("ftruncate failed")),
|
||||
)
|
||||
|
||||
with pytest.raises(OSError, match="ftruncate failed"):
|
||||
SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=4,
|
||||
rank=0,
|
||||
kv_bytes_per_block=PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
)
|
||||
|
||||
mock_unlink.assert_called_once_with(mmap_path)
|
||||
mock_close.assert_called_once_with(9999)
|
||||
|
||||
@@ -6,6 +6,9 @@ import time
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
check_shm_free_space,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -62,21 +65,37 @@ class SharedOffloadRegion:
|
||||
# exclusive upper bound for this worker's area within each row
|
||||
self._worker_area_end = (rank + 1) * cpu_page_size
|
||||
try:
|
||||
# Exclusive create — only one worker succeeds
|
||||
self.fd: int | None = os.open(
|
||||
self.mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600
|
||||
)
|
||||
os.ftruncate(self.fd, self.total_size_bytes)
|
||||
except FileExistsError:
|
||||
# Joiner path — another worker won O_EXCL. Reopen and wait
|
||||
# for the file to reach expected size.
|
||||
self.fd = os.open(self.mmap_path, os.O_RDWR)
|
||||
try:
|
||||
_wait_for_file_size(self.fd, self.total_size_bytes)
|
||||
except (TimeoutError, OSError):
|
||||
os.close(self.fd)
|
||||
raise
|
||||
logger.info("Opened existing mmap file %s", self.mmap_path)
|
||||
else:
|
||||
# Creator path. We won O_EXCL, so we own the file: any
|
||||
# failure here must clean up so concurrent joiners don't
|
||||
# land on a 0-byte stub and spin in _wait_for_file_size
|
||||
# for the full 30 s timeout.
|
||||
try:
|
||||
check_shm_free_space(self.total_size_bytes)
|
||||
os.ftruncate(self.fd, self.total_size_bytes)
|
||||
except (RuntimeError, OSError):
|
||||
os.unlink(self.mmap_path)
|
||||
os.close(self.fd)
|
||||
raise
|
||||
self._creator = True
|
||||
logger.info(
|
||||
"Created mmap file %s (%.2f GB)",
|
||||
self.mmap_path,
|
||||
self.total_size_bytes / 1e9,
|
||||
)
|
||||
except FileExistsError:
|
||||
self.fd = os.open(self.mmap_path, os.O_RDWR)
|
||||
_wait_for_file_size(self.fd, self.total_size_bytes)
|
||||
logger.info("Opened existing mmap file %s", self.mmap_path)
|
||||
|
||||
self.mmap_obj: mmap.mmap | None = mmap.mmap(
|
||||
self.fd,
|
||||
|
||||
Reference in New Issue
Block a user