[Bugfix] Fail fast with a clear error when CPU offload region exceeds available space (#50358)

Signed-off-by: Alex <jihui.huang@daocloud.io>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
AlexHuang
2026-08-05 20:59:55 +03:00
committed by GitHub
co-authored by OpenAI Codex Or Ozeri
parent e8586b2878
commit 9833aa53d5
2 changed files with 95 additions and 6 deletions
@@ -6,6 +6,9 @@ import time
import torch
from vllm.distributed.device_communicators.shm_broadcast import (
check_shm_free_space,
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
@@ -62,21 +65,37 @@ class SharedOffloadRegion:
# exclusive upper bound for this worker's area within each row
self._worker_area_end = (rank + 1) * cpu_page_size
try:
# Exclusive create — only one worker succeeds
self.fd: int | None = os.open(
self.mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600
)
os.ftruncate(self.fd, self.total_size_bytes)
except FileExistsError:
# Joiner path — another worker won O_EXCL. Reopen and wait
# for the file to reach expected size.
self.fd = os.open(self.mmap_path, os.O_RDWR)
try:
_wait_for_file_size(self.fd, self.total_size_bytes)
except (TimeoutError, OSError):
os.close(self.fd)
raise
logger.info("Opened existing mmap file %s", self.mmap_path)
else:
# Creator path. We won O_EXCL, so we own the file: any
# failure here must clean up so concurrent joiners don't
# land on a 0-byte stub and spin in _wait_for_file_size
# for the full 30 s timeout.
try:
check_shm_free_space(self.total_size_bytes)
os.ftruncate(self.fd, self.total_size_bytes)
except (RuntimeError, OSError):
os.unlink(self.mmap_path)
os.close(self.fd)
raise
self._creator = True
logger.info(
"Created mmap file %s (%.2f GB)",
self.mmap_path,
self.total_size_bytes / 1e9,
)
except FileExistsError:
self.fd = os.open(self.mmap_path, os.O_RDWR)
_wait_for_file_size(self.fd, self.total_size_bytes)
logger.info("Opened existing mmap file %s", self.mmap_path)
self.mmap_obj: mmap.mmap | None = mmap.mmap(
self.fd,