[Bugfix] Fail fast with a clear error when CPU offload region exceeds available space (#50358)
Signed-off-by: Alex <jihui.huang@daocloud.io> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
OpenAI Codex
Or Ozeri
parent
e8586b2878
commit
9833aa53d5
@@ -6,6 +6,9 @@ import time
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
check_shm_free_space,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -62,21 +65,37 @@ class SharedOffloadRegion:
|
||||
# exclusive upper bound for this worker's area within each row
|
||||
self._worker_area_end = (rank + 1) * cpu_page_size
|
||||
try:
|
||||
# Exclusive create — only one worker succeeds
|
||||
self.fd: int | None = os.open(
|
||||
self.mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600
|
||||
)
|
||||
os.ftruncate(self.fd, self.total_size_bytes)
|
||||
except FileExistsError:
|
||||
# Joiner path — another worker won O_EXCL. Reopen and wait
|
||||
# for the file to reach expected size.
|
||||
self.fd = os.open(self.mmap_path, os.O_RDWR)
|
||||
try:
|
||||
_wait_for_file_size(self.fd, self.total_size_bytes)
|
||||
except (TimeoutError, OSError):
|
||||
os.close(self.fd)
|
||||
raise
|
||||
logger.info("Opened existing mmap file %s", self.mmap_path)
|
||||
else:
|
||||
# Creator path. We won O_EXCL, so we own the file: any
|
||||
# failure here must clean up so concurrent joiners don't
|
||||
# land on a 0-byte stub and spin in _wait_for_file_size
|
||||
# for the full 30 s timeout.
|
||||
try:
|
||||
check_shm_free_space(self.total_size_bytes)
|
||||
os.ftruncate(self.fd, self.total_size_bytes)
|
||||
except (RuntimeError, OSError):
|
||||
os.unlink(self.mmap_path)
|
||||
os.close(self.fd)
|
||||
raise
|
||||
self._creator = True
|
||||
logger.info(
|
||||
"Created mmap file %s (%.2f GB)",
|
||||
self.mmap_path,
|
||||
self.total_size_bytes / 1e9,
|
||||
)
|
||||
except FileExistsError:
|
||||
self.fd = os.open(self.mmap_path, os.O_RDWR)
|
||||
_wait_for_file_size(self.fd, self.total_size_bytes)
|
||||
logger.info("Opened existing mmap file %s", self.mmap_path)
|
||||
|
||||
self.mmap_obj: mmap.mmap | None = mmap.mmap(
|
||||
self.fd,
|
||||
|
||||
Reference in New Issue
Block a user