From efad400092bac444608ffdac937431239696a06f Mon Sep 17 00:00:00 2001 From: Itay Etelis <92247226+Etelis@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:28:26 +0300 Subject: [PATCH] [HiSparse] Prefault the shared host pool in parallel (#60869) Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Codex --- .../cpu/test_shared_offload_region.py | 103 +++++++++++++----- tests/v1/worker/test_utils.py | 15 ++- vllm/v1/hisparse/runtime.py | 5 +- .../kv_offload/cpu/shared_offload_region.py | 87 ++++++--------- 4 files changed, 119 insertions(+), 91 deletions(-) diff --git a/tests/v1/kv_offload/cpu/test_shared_offload_region.py b/tests/v1/kv_offload/cpu/test_shared_offload_region.py index 502350ca47..ada31beac2 100644 --- a/tests/v1/kv_offload/cpu/test_shared_offload_region.py +++ b/tests/v1/kv_offload/cpu/test_shared_offload_region.py @@ -217,7 +217,7 @@ def _mp_barrier_construct_and_hold( kv_bytes_per_chunk=PAGE_SIZE if replicated else num_workers * PAGE_SIZE, cpu_page_size=PAGE_SIZE, barrier=lambda: barrier.wait(30), - populate_only_on_creator=replicated, + populate_shard=(rank, num_workers) if replicated else None, ) # The constructor's barrier precedes the creator's unlink. barrier.wait(30) @@ -538,13 +538,23 @@ def test_populate_shard_prefaults_only_its_share(iid, monkeypatch, shard, expect from vllm.v1.kv_offload.cpu import shared_offload_region as sor madvise_calls: list[tuple[int, int]] = [] - monkeypatch.setattr( - sor, - "_madvise_populate_write", - lambda mm, off, ln: madvise_calls.append((off, ln)), - ) - with _region(iid, num_chunks=3, num_workers=2, populate_shard=shard): + + def populate(mm, off, ln): + assert not os.path.exists(f"/dev/shm/vllm_offload_{iid}.mmap") + madvise_calls.append((off, ln)) + + monkeypatch.setattr(sor, "_madvise_populate_write", populate) + + def check_population(): + if barrier.call_count == 2: + assert madvise_calls == [(0, PAGE_SIZE), *expected] + + barrier = MagicMock(side_effect=check_population) + with _region( + iid, num_chunks=3, num_workers=2, populate_shard=shard, barrier=barrier + ): assert madvise_calls == [(0, PAGE_SIZE), *expected] + assert barrier.call_count == 2 def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch): @@ -1098,12 +1108,17 @@ def test_backing_file_unlinked_after_barrier(iid): _cleanup_file(path) -def test_barrier_failure_unlinks_creator_and_raises(iid): +@pytest.mark.parametrize("populate_shard", [None, (0, 1)]) +def test_barrier_failure_unlinks_creator_and_raises(iid, populate_shard): """A failed rendezvous must remove the creator's file and re-raise, not leave a stub that wedges the next start in _wait_for_file_size.""" path = f"/dev/shm/vllm_offload_{iid}.mmap" with pytest.raises(RuntimeError, match="peer died"): - _make_region(iid, barrier=MagicMock(side_effect=RuntimeError("peer died"))) + _make_region( + iid, + barrier=MagicMock(side_effect=RuntimeError("peer died")), + populate_shard=populate_shard, + ) assert not os.path.exists(path) @@ -1132,9 +1147,7 @@ def test_mp_barrier_unlinks_file_and_survives_sigkill(iid, replicated): assert len({r["inode"] for r in results}) == 1, "workers split onto two files" assert not any(r["path_exists"] for r in results) assert all(r["contents_match"] for r in results) - assert sum(r["populate_calls"] for r in results) == ( - 1 if replicated else num_workers - ) + assert sum(r["populate_calls"] for r in results) == num_workers assert not os.path.exists(path) finally: for p in procs: @@ -1146,23 +1159,40 @@ def test_mp_barrier_unlinks_file_and_survives_sigkill(iid, replicated): assert restarted._creator is True, "restart must be able to create anew" -@pytest.mark.parametrize("failure_phase", ["shm", "memory", "population"]) -def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch, failure_phase): +@pytest.mark.parametrize( + "failure_phase,populate_shard", + [("shm", None), ("memory", None)] + + [ + (phase, (0, 1)) + for phase in ("shm", "memory", "probe", "population", "population_view") + ], +) +def test_setup_failure_before_barrier_releases_peers( + iid, monkeypatch, failure_phase, populate_shard +): """A worker that dies before the rendezvous must still arrive at the barrier, or its peers block in the collective until it times out.""" - failure = MagicMock(side_effect=ValueError("Insufficient space")) + failure = MagicMock(side_effect=ValueError("setup failed")) memory_check = failure if failure_phase == "memory" else None if failure_phase == "shm": monkeypatch.setattr(region_module, "check_shm_free_space", failure) - elif failure_phase == "population": - monkeypatch.setattr(region_module, "_get_populate_write_fn", failure) + elif failure_phase in ("probe", "population", "population_view"): + + def populate(mm, offset, length): + failure(memoryview(mm) if failure_phase == "population_view" else None) + + monkeypatch.setattr( + region_module, + "_get_populate_write_fn", + failure if failure_phase == "probe" else lambda mm: populate, + ) path = f"/dev/shm/vllm_offload_{iid}.mmap" seen_at_barrier = [] barrier = MagicMock( side_effect=lambda: seen_at_barrier.append(os.path.exists(path)) ) - with pytest.raises(ValueError, match="Insufficient space"): + with pytest.raises(ValueError, match="setup failed"): SharedOffloadRegion( engine_id=iid, num_chunks=1, @@ -1171,11 +1201,15 @@ def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch, failure_p cpu_page_size=PAGE_SIZE, barrier=barrier, creator_memory_check=memory_check, - populate_only_on_creator=True, + populate_shard=populate_shard, ) - barrier.assert_called_once_with() - assert seen_at_barrier == [False] + assert barrier.call_count == (2 if populate_shard is not None else 1) + assert seen_at_barrier == ( + [False] * barrier.call_count + if failure_phase in ("shm", "memory") + else [True, False] + ) def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch): @@ -1191,18 +1225,29 @@ def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch): assert seen_at_barrier == [False], "file must be gone before peers release" -def test_barrier_release_failure_keeps_original_error(iid, monkeypatch): +@pytest.mark.parametrize("failure_phase", ["shm", "population"]) +def test_barrier_release_failure_keeps_original_error(iid, monkeypatch, failure_phase): """When releasing the peers fails too, the setup error that explains the failure must be the one that propagates.""" import vllm.v1.kv_offload.cpu.shared_offload_region as region - monkeypatch.setattr( - region, - "check_shm_free_space", - MagicMock(side_effect=RuntimeError("Insufficient space")), + failure = MagicMock(side_effect=RuntimeError("setup failed")) + if failure_phase == "shm": + monkeypatch.setattr(region, "check_shm_free_space", failure) + else: + monkeypatch.setattr(region, "_get_populate_write_fn", lambda mm: failure) + + barrier = MagicMock( + side_effect=[None, TimeoutError("barrier")] + if failure_phase == "population" + else TimeoutError("barrier") ) + with pytest.raises(RuntimeError, match="setup failed"): + _make_region( + iid, + barrier=barrier, + populate_shard=(0, 1) if failure_phase == "population" else None, + ) - with pytest.raises(RuntimeError, match="Insufficient space"): - _make_region(iid, barrier=MagicMock(side_effect=TimeoutError("barrier"))) - + assert barrier.call_count == (2 if failure_phase == "population" else 1) assert not os.path.exists(f"/dev/shm/vllm_offload_{iid}.mmap") diff --git a/tests/v1/worker/test_utils.py b/tests/v1/worker/test_utils.py index 84dd944e1d..a8f49c1b88 100644 --- a/tests/v1/worker/test_utils.py +++ b/tests/v1/worker/test_utils.py @@ -239,7 +239,7 @@ def test_hisparse_shares_host_pool_only_for_local_tp(monkeypatch): @pytest.mark.skip_global_cleanup def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch): page = mmap.PAGESIZE - tp_group = MagicMock() + tp_group = MagicMock(rank_in_group=1, world_size=2) monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", lambda: tp_group) class FakeSharedOffloadRegion: @@ -253,6 +253,7 @@ def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch): self.pinned_addresses = [] self.view_sizes = [] self.offset = 0 + kwargs["barrier"]() def create_next_canonical_view(self, size): self.view_sizes.append(size) @@ -280,16 +281,13 @@ def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch): pinned: list[torch.Tensor] = [] def pin_tensor(tensor): + tp_group.barrier.assert_called_once_with() pinned.append(tensor) monkeypatch.setattr(hisparse_runtime_module, "pin_tensor", pin_tensor) config = SimpleNamespace( instance_id="instance", - parallel_config=_hisparse_parallel_config( - tensor_parallel_size=1, - world_size=1, - distributed_executor_backend="uni", - ), + parallel_config=_hisparse_parallel_config(), ) pools, private_pools, region = hisparse_runtime_module.allocate_hisparse_host_pools( @@ -310,7 +308,7 @@ def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch): "cpu_page_size": 64, "barrier": tp_group.barrier, "creator_memory_check": hisparse_runtime_module.check_hisparse_host_memory, - "populate_only_on_creator": True, + "populate_shard": (1, 2), } assert region.view_sizes == [24, 40] assert [pool.shape for pool in pools] == [(24,), (40,)] @@ -328,7 +326,8 @@ def test_shared_host_pool_tracks_successful_registrations( ): """Only successful registrations may be unregistered on allocation failure.""" page = mmap.PAGESIZE - monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", MagicMock()) + tp_group = MagicMock(rank_in_group=0, world_size=2) + monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", lambda: tp_group) backing = torch.empty(3 * page, dtype=torch.uint8) region = MagicMock(base_tensor=backing, pinned_addresses=[], is_pinned=False) monkeypatch.setattr( diff --git a/vllm/v1/hisparse/runtime.py b/vllm/v1/hisparse/runtime.py index f7474b6bed..3cc45c680e 100644 --- a/vllm/v1/hisparse/runtime.py +++ b/vllm/v1/hisparse/runtime.py @@ -367,6 +367,7 @@ def allocate_hisparse_host_pools( None, ) + tp_group = get_tp_group() region = SharedOffloadRegion( engine_id=( f"hisparse_{vllm_config.instance_id}_" @@ -376,9 +377,9 @@ def allocate_hisparse_host_pools( rank=0, kv_bytes_per_chunk=num_blocks * host_block_stride, cpu_page_size=sum(tensor_sizes), - barrier=get_tp_group().barrier, + barrier=tp_group.barrier, creator_memory_check=check_hisparse_host_memory, - populate_only_on_creator=True, + populate_shard=(tp_group.rank_in_group, tp_group.world_size), ) try: for start, end in _hisparse_registration_ranges( diff --git a/vllm/v1/kv_offload/cpu/shared_offload_region.py b/vllm/v1/kv_offload/cpu/shared_offload_region.py index a9b753f1e3..6a7e26b4b8 100644 --- a/vllm/v1/kv_offload/cpu/shared_offload_region.py +++ b/vllm/v1/kv_offload/cpu/shared_offload_region.py @@ -81,10 +81,9 @@ class SharedOffloadRegion: the kernel reclaims the memory when the last worker exits, no matter how it exits; mappings taken before the unlink stay valid. - Creator-only population pre-faults the entire region before the barrier - and requires that barrier to keep joiners from using unpopulated pages. ``populate_shard=(index, count)`` pre-faults only this worker's contiguous - share, for layouts where every worker maps the same bytes. + share, for layouts where every worker maps the same bytes. The barrier + also waits for all shards to finish pre-faulting before returning. """ BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE @@ -99,11 +98,8 @@ class SharedOffloadRegion: barrier: Callable[[], None] | None = None, *, creator_memory_check: Callable[[int], None] | None = None, - populate_only_on_creator: bool = False, populate_shard: tuple[int, int] | None = None, ) -> None: - if populate_only_on_creator and barrier is None: - raise ValueError("Creator-only population requires a barrier.") self.page_size = mmap.PAGESIZE assert kv_bytes_per_chunk % self.page_size == 0 @@ -119,6 +115,7 @@ class SharedOffloadRegion: self._worker_offset = rank * cpu_page_size # exclusive upper bound for this worker's area within each row self._worker_area_end = (rank + 1) * cpu_page_size + barriers_remaining = 2 if populate_shard is not None else 1 try: try: self.fd: int | None = os.open( @@ -155,75 +152,61 @@ class SharedOffloadRegion: flags=mmap.MAP_SHARED, prot=mmap.PROT_READ | mmap.PROT_WRITE, ) - - if populate_only_on_creator and self._creator: + if barrier is not None: + barriers_remaining -= 1 + barrier() + if self._creator: + os.unlink(self.mmap_path) + self._creator = False + logger.info("Unlinked mmap file %s", self.mmap_path) + if populate_shard is not None: populate_write_fn = _get_populate_write_fn(self.mmap_obj) - populate_write_fn(self.mmap_obj, 0, self.total_size_bytes) + index, count = populate_shard + share = round_up(cdiv(self.total_size_bytes, count), self.page_size) + start = min(index * share, self.total_size_bytes) + if start < self.total_size_bytes: + length = min(share, self.total_size_bytes - start) + populate_write_fn(self.mmap_obj, start, length) + if barrier is not None: + barriers_remaining -= 1 + barrier() except Exception: if self._creator: with contextlib.suppress(FileNotFoundError): os.unlink(self.mmap_path) self._creator = False if hasattr(self, "mmap_obj") and self.mmap_obj is not None: - self.mmap_obj.close() + # The prefault traceback may retain an exported buffer. + with contextlib.suppress(BufferError): + self.mmap_obj.close() self.mmap_obj = None if hasattr(self, "fd") and self.fd is not None: os.close(self.fd) self.fd = None - # Peers block inside the barrier until the collective times out if - # we die before reaching it. Arrive anyway so every worker calls - # barrier() exactly once and they fail on their own errors instead - # of hanging; a failure here must not replace ours. + # Join remaining barriers so peers do not wait until the collective + # times out. A barrier failure must not replace the setup error. if barrier is not None: - try: - barrier() - except Exception: - logger.warning( - "Failed to release peers waiting at the mmap barrier", - exc_info=True, - ) + for _ in range(barriers_remaining): + try: + barrier() + except Exception: + logger.warning( + "Failed to release peers waiting at the mmap barrier", + exc_info=True, + ) raise - if barrier is not None: - # Every worker has mapped the file once the barrier releases, so - # its name is no longer needed and dropping it here means no exit - # path — including SIGKILL — can leak the file. - try: - barrier() - except Exception: - if self._creator: - with contextlib.suppress(FileNotFoundError): - os.unlink(self.mmap_path) - self._creator = False - self.mmap_obj.close() - os.close(self.fd) - self.mmap_obj = None - self.fd = None - raise - if self._creator: - os.unlink(self.mmap_path) - self._creator = False - logger.info("Unlinked mmap file %s", self.mmap_path) - self._base = torch.frombuffer(memoryview(self.mmap_obj), dtype=torch.int8) self._views: list[torch.Tensor] = [] self._canonical_offset = 0 self.is_pinned: bool = False self.pinned_addresses: list[int] = [] - if populate_only_on_creator: + if populate_shard is not None: return - populate_write_fn = _get_populate_write_fn(self.mmap_obj) - if populate_shard is not None: - index, count = populate_shard - share = round_up(cdiv(self.total_size_bytes, count), self.page_size) - start = min(index * share, self.total_size_bytes) - if start < self.total_size_bytes: - length = min(share, self.total_size_bytes - start) - populate_write_fn(self.mmap_obj, start, length) - elif rank is not None: + if rank is not None: # Populate only this worker's pages (one slot per chunk row). worker_offset = rank * cpu_page_size _t0 = time.perf_counter()