diff --git a/tests/v1/kv_offload/cpu/test_shared_offload_region.py b/tests/v1/kv_offload/cpu/test_shared_offload_region.py index 1cdb589ef1..1d76f60623 100644 --- a/tests/v1/kv_offload/cpu/test_shared_offload_region.py +++ b/tests/v1/kv_offload/cpu/test_shared_offload_region.py @@ -42,6 +42,7 @@ def _make_region( cpu_page_size: int = PAGE_SIZE, num_workers: int = 1, rank: int = 0, + barrier=None, ) -> SharedOffloadRegion: assert cpu_page_size % PAGE_SIZE == 0 return SharedOffloadRegion( @@ -50,6 +51,7 @@ def _make_region( rank=rank, kv_bytes_per_block=num_workers * cpu_page_size, cpu_page_size=cpu_page_size, + barrier=barrier, ) @@ -183,6 +185,40 @@ def _mp_race_construct_and_write( done_queue.put({"rank": rank, "error": repr(e)}) +def _mp_barrier_construct_and_hold( + engine_id: str, + rank: int, + num_workers: int, + barrier, + fill_value: int, + done_queue, +) -> None: + """Construct with a real cross-process barrier, write, then hold the + mapping (no cleanup) until the parent SIGKILLs this process.""" + try: + region = SharedOffloadRegion( + engine_id=engine_id, + num_blocks=2, + rank=rank, + kv_bytes_per_block=num_workers * PAGE_SIZE, + cpu_page_size=PAGE_SIZE, + barrier=lambda: barrier.wait(30), + ) + t = region.create_next_worker_view(PAGE_SIZE) + t[:, :] = fill_value + done_queue.put( + { + "rank": rank, + "error": None, + "inode": os.fstat(region.fd).st_ino, + "path_exists": os.path.exists(region.mmap_path), + } + ) + time.sleep(60) # hold the mapping until killed + except Exception as e: + done_queue.put({"rank": rank, "error": repr(e)}) + + @pytest.fixture def iid(): """Fresh instance ID for each test.""" @@ -840,3 +876,122 @@ def test_ftruncate_failure_cleans_up_creator(monkeypatch): mock_unlink.assert_called_once_with(mmap_path) mock_close.assert_called_once_with(9999) + + +# --------------------------------------------------------------------------- +# Unlink-after-barrier — leak immunity to hard kills +# --------------------------------------------------------------------------- + + +def test_backing_file_unlinked_after_barrier(iid): + """The file must exist until the barrier releases (late joiners need the + name) and be gone right after, so no exit path can leak it.""" + path = f"/dev/shm/vllm_offload_{iid}.mmap" + seen_at_barrier = [] + + region = _make_region( + iid, barrier=lambda: seen_at_barrier.append(os.path.exists(path)) + ) + try: + assert seen_at_barrier == [True], "file must exist during rendezvous" + assert not os.path.exists(path), "name must be dropped after the barrier" + assert region._creator is False, "nothing left for cleanup() to unlink" + t = region.create_next_worker_view(PAGE_SIZE) + t[:, :] = 7 + assert memoryview(region.mmap_obj)[0] == 7, "mapping must stay valid" + del t + finally: + region.cleanup() + _cleanup_file(path) + + +def test_barrier_failure_unlinks_creator_and_raises(iid): + """A failed rendezvous must remove the creator's file and re-raise, not + leave a stub that wedges the next start in _wait_for_file_size.""" + path = f"/dev/shm/vllm_offload_{iid}.mmap" + with pytest.raises(RuntimeError, match="peer died"): + _make_region(iid, barrier=MagicMock(side_effect=RuntimeError("peer died"))) + assert not os.path.exists(path) + + +def test_mp_barrier_unlinks_file_and_survives_sigkill(iid): + """With a real cross-process barrier every worker maps one shared inode, + the name is gone while they run, and SIGKILL leaks nothing.""" + num_workers = 2 + path = f"/dev/shm/vllm_offload_{iid}.mmap" + ctx = get_mp_context() + barrier = ctx.Barrier(num_workers) + done_queue = ctx.Queue() + procs = [ + ctx.Process( + target=_mp_barrier_construct_and_hold, + args=(iid, rank, num_workers, barrier, rank + 1, done_queue), + ) + for rank in range(num_workers) + ] + try: + for p in procs: + p.start() + results = [done_queue.get(timeout=30) for _ in range(num_workers)] + for r in results: + assert r["error"] is None, f"rank {r['rank']}: {r['error']}" + assert len({r["inode"] for r in results}) == 1, "workers split onto two files" + assert not any(r["path_exists"] for r in results) + assert not os.path.exists(path) + finally: + for p in procs: + p.kill() # SIGKILL: no cleanup() runs + p.join(timeout=10) + + assert not os.path.exists(path), "SIGKILL must not leak the file" + with _region(iid) as restarted: + assert restarted._creator is True, "restart must be able to create anew" + + +def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch): + """A worker that dies before the rendezvous must still arrive at the + barrier, or its peers block in the collective until it times out.""" + import vllm.v1.kv_offload.cpu.shared_offload_region as region + + monkeypatch.setattr( + region, + "check_shm_free_space", + MagicMock(side_effect=RuntimeError("Insufficient space")), + ) + barrier = MagicMock() + + with pytest.raises(RuntimeError, match="Insufficient space"): + _make_region(iid, barrier=barrier) + + barrier.assert_called_once_with() + assert not os.path.exists(f"/dev/shm/vllm_offload_{iid}.mmap") + + +def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch): + """A creator that fails after sizing the file must drop it before arriving + at the barrier, so the next start does not land on a stale file.""" + path = f"/dev/shm/vllm_offload_{iid}.mmap" + monkeypatch.setattr("mmap.mmap", MagicMock(side_effect=OSError("mmap"))) + seen_at_barrier = [] + + with pytest.raises(OSError, match="mmap"): + _make_region(iid, barrier=lambda: seen_at_barrier.append(os.path.exists(path))) + + assert seen_at_barrier == [False], "file must be gone before peers release" + + +def test_barrier_release_failure_keeps_original_error(iid, monkeypatch): + """When releasing the peers fails too, the setup error that explains the + failure must be the one that propagates.""" + import vllm.v1.kv_offload.cpu.shared_offload_region as region + + monkeypatch.setattr( + region, + "check_shm_free_space", + MagicMock(side_effect=RuntimeError("Insufficient space")), + ) + + with pytest.raises(RuntimeError, match="Insufficient space"): + _make_region(iid, barrier=MagicMock(side_effect=TimeoutError("barrier"))) + + assert not os.path.exists(f"/dev/shm/vllm_offload_{iid}.mmap") diff --git a/vllm/v1/kv_offload/cpu/shared_offload_region.py b/vllm/v1/kv_offload/cpu/shared_offload_region.py index 75bea4e2e0..33639aab45 100644 --- a/vllm/v1/kv_offload/cpu/shared_offload_region.py +++ b/vllm/v1/kv_offload/cpu/shared_offload_region.py @@ -71,7 +71,10 @@ class SharedOffloadRegion: the rest open the existing file and wait until it reaches the expected size. Each worker then mmap()s the full file. - File path: /dev/shm/vllm_offload_{engine_id}.mmap + File path: /dev/shm/vllm_offload_{engine_id}.mmap. When a barrier is + given, the path is unlinked once every worker has mapped the file, so + the kernel reclaims the memory when the last worker exits, no matter + how it exits; mappings taken before the unlink stay valid. """ BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE @@ -83,6 +86,7 @@ class SharedOffloadRegion: rank: int | None, kv_bytes_per_block: int, cpu_page_size: int, + barrier: Callable[[], None] | None = None, ) -> None: self.page_size = mmap.PAGESIZE assert kv_bytes_per_block % self.page_size == 0 @@ -100,44 +104,80 @@ class SharedOffloadRegion: # exclusive upper bound for this worker's area within each row self._worker_area_end = (rank + 1) * cpu_page_size try: - self.fd: int | None = os.open( - self.mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600 - ) - except FileExistsError: - # Joiner path — another worker won O_EXCL. Reopen and wait - # for the file to reach expected size. - self.fd = os.open(self.mmap_path, os.O_RDWR) try: - _wait_for_file_size(self.fd, self.total_size_bytes) - except (TimeoutError, OSError): - os.close(self.fd) - raise - logger.info("Opened existing mmap file %s", self.mmap_path) - else: - # Creator path. We won O_EXCL, so we own the file: any - # failure here must clean up so concurrent joiners don't - # land on a 0-byte stub and spin in _wait_for_file_size - # for the full 30 s timeout. - try: - check_shm_free_space(self.total_size_bytes) - os.ftruncate(self.fd, self.total_size_bytes) - except (RuntimeError, OSError): - os.unlink(self.mmap_path) - os.close(self.fd) - raise - self._creator = True - logger.info( - "Created mmap file %s (%.2f GB)", - self.mmap_path, - self.total_size_bytes / 1e9, - ) + self.fd: int | None = os.open( + self.mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600 + ) + except FileExistsError: + # Joiner path — another worker won O_EXCL. Reopen and wait + # for the file to reach expected size. + self.fd = os.open(self.mmap_path, os.O_RDWR) + try: + _wait_for_file_size(self.fd, self.total_size_bytes) + except (TimeoutError, OSError): + os.close(self.fd) + raise + logger.info("Opened existing mmap file %s", self.mmap_path) + else: + # Creator path. We won O_EXCL, so we own the file: any + # failure here must clean up so concurrent joiners don't + # land on a 0-byte stub and spin in _wait_for_file_size + # for the full 30 s timeout. + try: + check_shm_free_space(self.total_size_bytes) + os.ftruncate(self.fd, self.total_size_bytes) + except (RuntimeError, OSError): + os.unlink(self.mmap_path) + os.close(self.fd) + raise + self._creator = True + logger.info( + "Created mmap file %s (%.2f GB)", + self.mmap_path, + self.total_size_bytes / 1e9, + ) - self.mmap_obj: mmap.mmap | None = mmap.mmap( - self.fd, - self.total_size_bytes, - flags=mmap.MAP_SHARED, - prot=mmap.PROT_READ | mmap.PROT_WRITE, - ) + self.mmap_obj: mmap.mmap | None = mmap.mmap( + self.fd, + self.total_size_bytes, + flags=mmap.MAP_SHARED, + prot=mmap.PROT_READ | mmap.PROT_WRITE, + ) + except Exception: + if self._creator: + os.unlink(self.mmap_path) + self._creator = False + # Peers block inside the barrier until the collective times out if + # we die before reaching it. Arrive anyway so every worker calls + # barrier() exactly once and they fail on their own errors instead + # of hanging; a failure here must not replace ours. + if barrier is not None: + try: + barrier() + except Exception: + logger.warning( + "Failed to release peers waiting at the mmap barrier", + exc_info=True, + ) + raise + + if barrier is not None: + # Every worker has mapped the file once the barrier releases, so + # its name is no longer needed and dropping it here means no exit + # path — including SIGKILL — can leak the file. + try: + barrier() + except Exception: + if self._creator: + os.unlink(self.mmap_path) + self._creator = False + self.mmap_obj.close() + os.close(self.fd) + raise + if self._creator: + os.unlink(self.mmap_path) + self._creator = False + logger.info("Unlinked mmap file %s", self.mmap_path) populate_write_fn = _get_populate_write_fn(self.mmap_obj) diff --git a/vllm/v1/kv_offload/cpu/spec.py b/vllm/v1/kv_offload/cpu/spec.py index 6915a821d5..0a1ac6d6a0 100644 --- a/vllm/v1/kv_offload/cpu/spec.py +++ b/vllm/v1/kv_offload/cpu/spec.py @@ -24,6 +24,23 @@ from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion +def _all_workers_barrier() -> None: + """Block until every worker rank has reached this point (gloo cpu group). + + A superset of the node-local mmap openers suffices: once the barrier + releases, every worker sharing the region file has mapped it.""" + from vllm.distributed.parallel_state import ( + get_inner_dp_world_group, + get_world_group, + ) + + try: + group = get_inner_dp_world_group() + except AssertionError: + group = get_world_group() + group.barrier() + + class CPUOffloadingSpec(OffloadingSpec): BLOCK_SIZE_ALIGNMENT = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT @@ -165,6 +182,7 @@ class CPUOffloadingSpec(OffloadingSpec): rank=rank, kv_bytes_per_block=self.kv_bytes_per_chunk, cpu_page_size=self.cpu_page_size_per_worker, + barrier=_all_workers_barrier, ) try: return CPUOffloadingWorker(