[Bugfix][KV Offload] Unlink /dev/shm region after all workers map it (barrier variant of #51317) (#52596)
Signed-off-by: Itay Etelis <itay.etelis@ibm.com> Co-authored-by: Itay Etelis <itay.etelis@ibm.com> Co-authored-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
co-authored by
Itay Etelis
NickLucche
parent
e0d27040dd
commit
4c58a0c398
@@ -42,6 +42,7 @@ def _make_region(
|
||||
cpu_page_size: int = PAGE_SIZE,
|
||||
num_workers: int = 1,
|
||||
rank: int = 0,
|
||||
barrier=None,
|
||||
) -> SharedOffloadRegion:
|
||||
assert cpu_page_size % PAGE_SIZE == 0
|
||||
return SharedOffloadRegion(
|
||||
@@ -50,6 +51,7 @@ def _make_region(
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
||||
cpu_page_size=cpu_page_size,
|
||||
barrier=barrier,
|
||||
)
|
||||
|
||||
|
||||
@@ -183,6 +185,40 @@ def _mp_race_construct_and_write(
|
||||
done_queue.put({"rank": rank, "error": repr(e)})
|
||||
|
||||
|
||||
def _mp_barrier_construct_and_hold(
|
||||
engine_id: str,
|
||||
rank: int,
|
||||
num_workers: int,
|
||||
barrier,
|
||||
fill_value: int,
|
||||
done_queue,
|
||||
) -> None:
|
||||
"""Construct with a real cross-process barrier, write, then hold the
|
||||
mapping (no cleanup) until the parent SIGKILLs this process."""
|
||||
try:
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=2,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
barrier=lambda: barrier.wait(30),
|
||||
)
|
||||
t = region.create_next_worker_view(PAGE_SIZE)
|
||||
t[:, :] = fill_value
|
||||
done_queue.put(
|
||||
{
|
||||
"rank": rank,
|
||||
"error": None,
|
||||
"inode": os.fstat(region.fd).st_ino,
|
||||
"path_exists": os.path.exists(region.mmap_path),
|
||||
}
|
||||
)
|
||||
time.sleep(60) # hold the mapping until killed
|
||||
except Exception as e:
|
||||
done_queue.put({"rank": rank, "error": repr(e)})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def iid():
|
||||
"""Fresh instance ID for each test."""
|
||||
@@ -840,3 +876,122 @@ def test_ftruncate_failure_cleans_up_creator(monkeypatch):
|
||||
|
||||
mock_unlink.assert_called_once_with(mmap_path)
|
||||
mock_close.assert_called_once_with(9999)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unlink-after-barrier — leak immunity to hard kills
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_backing_file_unlinked_after_barrier(iid):
|
||||
"""The file must exist until the barrier releases (late joiners need the
|
||||
name) and be gone right after, so no exit path can leak it."""
|
||||
path = f"/dev/shm/vllm_offload_{iid}.mmap"
|
||||
seen_at_barrier = []
|
||||
|
||||
region = _make_region(
|
||||
iid, barrier=lambda: seen_at_barrier.append(os.path.exists(path))
|
||||
)
|
||||
try:
|
||||
assert seen_at_barrier == [True], "file must exist during rendezvous"
|
||||
assert not os.path.exists(path), "name must be dropped after the barrier"
|
||||
assert region._creator is False, "nothing left for cleanup() to unlink"
|
||||
t = region.create_next_worker_view(PAGE_SIZE)
|
||||
t[:, :] = 7
|
||||
assert memoryview(region.mmap_obj)[0] == 7, "mapping must stay valid"
|
||||
del t
|
||||
finally:
|
||||
region.cleanup()
|
||||
_cleanup_file(path)
|
||||
|
||||
|
||||
def test_barrier_failure_unlinks_creator_and_raises(iid):
|
||||
"""A failed rendezvous must remove the creator's file and re-raise, not
|
||||
leave a stub that wedges the next start in _wait_for_file_size."""
|
||||
path = f"/dev/shm/vllm_offload_{iid}.mmap"
|
||||
with pytest.raises(RuntimeError, match="peer died"):
|
||||
_make_region(iid, barrier=MagicMock(side_effect=RuntimeError("peer died")))
|
||||
assert not os.path.exists(path)
|
||||
|
||||
|
||||
def test_mp_barrier_unlinks_file_and_survives_sigkill(iid):
|
||||
"""With a real cross-process barrier every worker maps one shared inode,
|
||||
the name is gone while they run, and SIGKILL leaks nothing."""
|
||||
num_workers = 2
|
||||
path = f"/dev/shm/vllm_offload_{iid}.mmap"
|
||||
ctx = get_mp_context()
|
||||
barrier = ctx.Barrier(num_workers)
|
||||
done_queue = ctx.Queue()
|
||||
procs = [
|
||||
ctx.Process(
|
||||
target=_mp_barrier_construct_and_hold,
|
||||
args=(iid, rank, num_workers, barrier, rank + 1, done_queue),
|
||||
)
|
||||
for rank in range(num_workers)
|
||||
]
|
||||
try:
|
||||
for p in procs:
|
||||
p.start()
|
||||
results = [done_queue.get(timeout=30) for _ in range(num_workers)]
|
||||
for r in results:
|
||||
assert r["error"] is None, f"rank {r['rank']}: {r['error']}"
|
||||
assert len({r["inode"] for r in results}) == 1, "workers split onto two files"
|
||||
assert not any(r["path_exists"] for r in results)
|
||||
assert not os.path.exists(path)
|
||||
finally:
|
||||
for p in procs:
|
||||
p.kill() # SIGKILL: no cleanup() runs
|
||||
p.join(timeout=10)
|
||||
|
||||
assert not os.path.exists(path), "SIGKILL must not leak the file"
|
||||
with _region(iid) as restarted:
|
||||
assert restarted._creator is True, "restart must be able to create anew"
|
||||
|
||||
|
||||
def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch):
|
||||
"""A worker that dies before the rendezvous must still arrive at the
|
||||
barrier, or its peers block in the collective until it times out."""
|
||||
import vllm.v1.kv_offload.cpu.shared_offload_region as region
|
||||
|
||||
monkeypatch.setattr(
|
||||
region,
|
||||
"check_shm_free_space",
|
||||
MagicMock(side_effect=RuntimeError("Insufficient space")),
|
||||
)
|
||||
barrier = MagicMock()
|
||||
|
||||
with pytest.raises(RuntimeError, match="Insufficient space"):
|
||||
_make_region(iid, barrier=barrier)
|
||||
|
||||
barrier.assert_called_once_with()
|
||||
assert not os.path.exists(f"/dev/shm/vllm_offload_{iid}.mmap")
|
||||
|
||||
|
||||
def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch):
|
||||
"""A creator that fails after sizing the file must drop it before arriving
|
||||
at the barrier, so the next start does not land on a stale file."""
|
||||
path = f"/dev/shm/vllm_offload_{iid}.mmap"
|
||||
monkeypatch.setattr("mmap.mmap", MagicMock(side_effect=OSError("mmap")))
|
||||
seen_at_barrier = []
|
||||
|
||||
with pytest.raises(OSError, match="mmap"):
|
||||
_make_region(iid, barrier=lambda: seen_at_barrier.append(os.path.exists(path)))
|
||||
|
||||
assert seen_at_barrier == [False], "file must be gone before peers release"
|
||||
|
||||
|
||||
def test_barrier_release_failure_keeps_original_error(iid, monkeypatch):
|
||||
"""When releasing the peers fails too, the setup error that explains the
|
||||
failure must be the one that propagates."""
|
||||
import vllm.v1.kv_offload.cpu.shared_offload_region as region
|
||||
|
||||
monkeypatch.setattr(
|
||||
region,
|
||||
"check_shm_free_space",
|
||||
MagicMock(side_effect=RuntimeError("Insufficient space")),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Insufficient space"):
|
||||
_make_region(iid, barrier=MagicMock(side_effect=TimeoutError("barrier")))
|
||||
|
||||
assert not os.path.exists(f"/dev/shm/vllm_offload_{iid}.mmap")
|
||||
|
||||
@@ -71,7 +71,10 @@ class SharedOffloadRegion:
|
||||
the rest open the existing file and wait until it reaches the expected
|
||||
size. Each worker then mmap()s the full file.
|
||||
|
||||
File path: /dev/shm/vllm_offload_{engine_id}.mmap
|
||||
File path: /dev/shm/vllm_offload_{engine_id}.mmap. When a barrier is
|
||||
given, the path is unlinked once every worker has mapped the file, so
|
||||
the kernel reclaims the memory when the last worker exits, no matter
|
||||
how it exits; mappings taken before the unlink stay valid.
|
||||
"""
|
||||
|
||||
BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE
|
||||
@@ -83,6 +86,7 @@ class SharedOffloadRegion:
|
||||
rank: int | None,
|
||||
kv_bytes_per_block: int,
|
||||
cpu_page_size: int,
|
||||
barrier: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
self.page_size = mmap.PAGESIZE
|
||||
assert kv_bytes_per_block % self.page_size == 0
|
||||
@@ -100,44 +104,80 @@ class SharedOffloadRegion:
|
||||
# exclusive upper bound for this worker's area within each row
|
||||
self._worker_area_end = (rank + 1) * cpu_page_size
|
||||
try:
|
||||
self.fd: int | None = os.open(
|
||||
self.mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600
|
||||
)
|
||||
except FileExistsError:
|
||||
# Joiner path — another worker won O_EXCL. Reopen and wait
|
||||
# for the file to reach expected size.
|
||||
self.fd = os.open(self.mmap_path, os.O_RDWR)
|
||||
try:
|
||||
_wait_for_file_size(self.fd, self.total_size_bytes)
|
||||
except (TimeoutError, OSError):
|
||||
os.close(self.fd)
|
||||
raise
|
||||
logger.info("Opened existing mmap file %s", self.mmap_path)
|
||||
else:
|
||||
# Creator path. We won O_EXCL, so we own the file: any
|
||||
# failure here must clean up so concurrent joiners don't
|
||||
# land on a 0-byte stub and spin in _wait_for_file_size
|
||||
# for the full 30 s timeout.
|
||||
try:
|
||||
check_shm_free_space(self.total_size_bytes)
|
||||
os.ftruncate(self.fd, self.total_size_bytes)
|
||||
except (RuntimeError, OSError):
|
||||
os.unlink(self.mmap_path)
|
||||
os.close(self.fd)
|
||||
raise
|
||||
self._creator = True
|
||||
logger.info(
|
||||
"Created mmap file %s (%.2f GB)",
|
||||
self.mmap_path,
|
||||
self.total_size_bytes / 1e9,
|
||||
)
|
||||
self.fd: int | None = os.open(
|
||||
self.mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600
|
||||
)
|
||||
except FileExistsError:
|
||||
# Joiner path — another worker won O_EXCL. Reopen and wait
|
||||
# for the file to reach expected size.
|
||||
self.fd = os.open(self.mmap_path, os.O_RDWR)
|
||||
try:
|
||||
_wait_for_file_size(self.fd, self.total_size_bytes)
|
||||
except (TimeoutError, OSError):
|
||||
os.close(self.fd)
|
||||
raise
|
||||
logger.info("Opened existing mmap file %s", self.mmap_path)
|
||||
else:
|
||||
# Creator path. We won O_EXCL, so we own the file: any
|
||||
# failure here must clean up so concurrent joiners don't
|
||||
# land on a 0-byte stub and spin in _wait_for_file_size
|
||||
# for the full 30 s timeout.
|
||||
try:
|
||||
check_shm_free_space(self.total_size_bytes)
|
||||
os.ftruncate(self.fd, self.total_size_bytes)
|
||||
except (RuntimeError, OSError):
|
||||
os.unlink(self.mmap_path)
|
||||
os.close(self.fd)
|
||||
raise
|
||||
self._creator = True
|
||||
logger.info(
|
||||
"Created mmap file %s (%.2f GB)",
|
||||
self.mmap_path,
|
||||
self.total_size_bytes / 1e9,
|
||||
)
|
||||
|
||||
self.mmap_obj: mmap.mmap | None = mmap.mmap(
|
||||
self.fd,
|
||||
self.total_size_bytes,
|
||||
flags=mmap.MAP_SHARED,
|
||||
prot=mmap.PROT_READ | mmap.PROT_WRITE,
|
||||
)
|
||||
self.mmap_obj: mmap.mmap | None = mmap.mmap(
|
||||
self.fd,
|
||||
self.total_size_bytes,
|
||||
flags=mmap.MAP_SHARED,
|
||||
prot=mmap.PROT_READ | mmap.PROT_WRITE,
|
||||
)
|
||||
except Exception:
|
||||
if self._creator:
|
||||
os.unlink(self.mmap_path)
|
||||
self._creator = False
|
||||
# Peers block inside the barrier until the collective times out if
|
||||
# we die before reaching it. Arrive anyway so every worker calls
|
||||
# barrier() exactly once and they fail on their own errors instead
|
||||
# of hanging; a failure here must not replace ours.
|
||||
if barrier is not None:
|
||||
try:
|
||||
barrier()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to release peers waiting at the mmap barrier",
|
||||
exc_info=True,
|
||||
)
|
||||
raise
|
||||
|
||||
if barrier is not None:
|
||||
# Every worker has mapped the file once the barrier releases, so
|
||||
# its name is no longer needed and dropping it here means no exit
|
||||
# path — including SIGKILL — can leak the file.
|
||||
try:
|
||||
barrier()
|
||||
except Exception:
|
||||
if self._creator:
|
||||
os.unlink(self.mmap_path)
|
||||
self._creator = False
|
||||
self.mmap_obj.close()
|
||||
os.close(self.fd)
|
||||
raise
|
||||
if self._creator:
|
||||
os.unlink(self.mmap_path)
|
||||
self._creator = False
|
||||
logger.info("Unlinked mmap file %s", self.mmap_path)
|
||||
|
||||
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
|
||||
|
||||
|
||||
@@ -24,6 +24,23 @@ from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
|
||||
|
||||
def _all_workers_barrier() -> None:
|
||||
"""Block until every worker rank has reached this point (gloo cpu group).
|
||||
|
||||
A superset of the node-local mmap openers suffices: once the barrier
|
||||
releases, every worker sharing the region file has mapped it."""
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_inner_dp_world_group,
|
||||
get_world_group,
|
||||
)
|
||||
|
||||
try:
|
||||
group = get_inner_dp_world_group()
|
||||
except AssertionError:
|
||||
group = get_world_group()
|
||||
group.barrier()
|
||||
|
||||
|
||||
class CPUOffloadingSpec(OffloadingSpec):
|
||||
BLOCK_SIZE_ALIGNMENT = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT
|
||||
|
||||
@@ -165,6 +182,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
rank=rank,
|
||||
kv_bytes_per_block=self.kv_bytes_per_chunk,
|
||||
cpu_page_size=self.cpu_page_size_per_worker,
|
||||
barrier=_all_workers_barrier,
|
||||
)
|
||||
try:
|
||||
return CPUOffloadingWorker(
|
||||
|
||||
Reference in New Issue
Block a user