[HiSparse] Prefault the shared host pool in parallel (#60869)

Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Itay Etelis
2026-10-14 10:28:26 -04:00
committed by GitHub
co-authored by Codex
parent 02653fa22c
commit efad400092
4 changed files with 119 additions and 91 deletions
@@ -217,7 +217,7 @@ def _mp_barrier_construct_and_hold(
kv_bytes_per_chunk=PAGE_SIZE if replicated else num_workers * PAGE_SIZE,
cpu_page_size=PAGE_SIZE,
barrier=lambda: barrier.wait(30),
populate_only_on_creator=replicated,
populate_shard=(rank, num_workers) if replicated else None,
)
# The constructor's barrier precedes the creator's unlink.
barrier.wait(30)
@@ -538,13 +538,23 @@ def test_populate_shard_prefaults_only_its_share(iid, monkeypatch, shard, expect
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
madvise_calls: list[tuple[int, int]] = []
monkeypatch.setattr(
sor,
"_madvise_populate_write",
lambda mm, off, ln: madvise_calls.append((off, ln)),
)
with _region(iid, num_chunks=3, num_workers=2, populate_shard=shard):
def populate(mm, off, ln):
assert not os.path.exists(f"/dev/shm/vllm_offload_{iid}.mmap")
madvise_calls.append((off, ln))
monkeypatch.setattr(sor, "_madvise_populate_write", populate)
def check_population():
if barrier.call_count == 2:
assert madvise_calls == [(0, PAGE_SIZE), *expected]
barrier = MagicMock(side_effect=check_population)
with _region(
iid, num_chunks=3, num_workers=2, populate_shard=shard, barrier=barrier
):
assert madvise_calls == [(0, PAGE_SIZE), *expected]
assert barrier.call_count == 2
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
@@ -1098,12 +1108,17 @@ def test_backing_file_unlinked_after_barrier(iid):
_cleanup_file(path)
def test_barrier_failure_unlinks_creator_and_raises(iid):
@pytest.mark.parametrize("populate_shard", [None, (0, 1)])
def test_barrier_failure_unlinks_creator_and_raises(iid, populate_shard):
"""A failed rendezvous must remove the creator's file and re-raise, not
leave a stub that wedges the next start in _wait_for_file_size."""
path = f"/dev/shm/vllm_offload_{iid}.mmap"
with pytest.raises(RuntimeError, match="peer died"):
_make_region(iid, barrier=MagicMock(side_effect=RuntimeError("peer died")))
_make_region(
iid,
barrier=MagicMock(side_effect=RuntimeError("peer died")),
populate_shard=populate_shard,
)
assert not os.path.exists(path)
@@ -1132,9 +1147,7 @@ def test_mp_barrier_unlinks_file_and_survives_sigkill(iid, replicated):
assert len({r["inode"] for r in results}) == 1, "workers split onto two files"
assert not any(r["path_exists"] for r in results)
assert all(r["contents_match"] for r in results)
assert sum(r["populate_calls"] for r in results) == (
1 if replicated else num_workers
)
assert sum(r["populate_calls"] for r in results) == num_workers
assert not os.path.exists(path)
finally:
for p in procs:
@@ -1146,23 +1159,40 @@ def test_mp_barrier_unlinks_file_and_survives_sigkill(iid, replicated):
assert restarted._creator is True, "restart must be able to create anew"
@pytest.mark.parametrize("failure_phase", ["shm", "memory", "population"])
def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch, failure_phase):
@pytest.mark.parametrize(
"failure_phase,populate_shard",
[("shm", None), ("memory", None)]
+ [
(phase, (0, 1))
for phase in ("shm", "memory", "probe", "population", "population_view")
],
)
def test_setup_failure_before_barrier_releases_peers(
iid, monkeypatch, failure_phase, populate_shard
):
"""A worker that dies before the rendezvous must still arrive at the
barrier, or its peers block in the collective until it times out."""
failure = MagicMock(side_effect=ValueError("Insufficient space"))
failure = MagicMock(side_effect=ValueError("setup failed"))
memory_check = failure if failure_phase == "memory" else None
if failure_phase == "shm":
monkeypatch.setattr(region_module, "check_shm_free_space", failure)
elif failure_phase == "population":
monkeypatch.setattr(region_module, "_get_populate_write_fn", failure)
elif failure_phase in ("probe", "population", "population_view"):
def populate(mm, offset, length):
failure(memoryview(mm) if failure_phase == "population_view" else None)
monkeypatch.setattr(
region_module,
"_get_populate_write_fn",
failure if failure_phase == "probe" else lambda mm: populate,
)
path = f"/dev/shm/vllm_offload_{iid}.mmap"
seen_at_barrier = []
barrier = MagicMock(
side_effect=lambda: seen_at_barrier.append(os.path.exists(path))
)
with pytest.raises(ValueError, match="Insufficient space"):
with pytest.raises(ValueError, match="setup failed"):
SharedOffloadRegion(
engine_id=iid,
num_chunks=1,
@@ -1171,11 +1201,15 @@ def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch, failure_p
cpu_page_size=PAGE_SIZE,
barrier=barrier,
creator_memory_check=memory_check,
populate_only_on_creator=True,
populate_shard=populate_shard,
)
barrier.assert_called_once_with()
assert seen_at_barrier == [False]
assert barrier.call_count == (2 if populate_shard is not None else 1)
assert seen_at_barrier == (
[False] * barrier.call_count
if failure_phase in ("shm", "memory")
else [True, False]
)
def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch):
@@ -1191,18 +1225,29 @@ def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch):
assert seen_at_barrier == [False], "file must be gone before peers release"
def test_barrier_release_failure_keeps_original_error(iid, monkeypatch):
@pytest.mark.parametrize("failure_phase", ["shm", "population"])
def test_barrier_release_failure_keeps_original_error(iid, monkeypatch, failure_phase):
"""When releasing the peers fails too, the setup error that explains the
failure must be the one that propagates."""
import vllm.v1.kv_offload.cpu.shared_offload_region as region
monkeypatch.setattr(
region,
"check_shm_free_space",
MagicMock(side_effect=RuntimeError("Insufficient space")),
failure = MagicMock(side_effect=RuntimeError("setup failed"))
if failure_phase == "shm":
monkeypatch.setattr(region, "check_shm_free_space", failure)
else:
monkeypatch.setattr(region, "_get_populate_write_fn", lambda mm: failure)
barrier = MagicMock(
side_effect=[None, TimeoutError("barrier")]
if failure_phase == "population"
else TimeoutError("barrier")
)
with pytest.raises(RuntimeError, match="setup failed"):
_make_region(
iid,
barrier=barrier,
populate_shard=(0, 1) if failure_phase == "population" else None,
)
with pytest.raises(RuntimeError, match="Insufficient space"):
_make_region(iid, barrier=MagicMock(side_effect=TimeoutError("barrier")))
assert barrier.call_count == (2 if failure_phase == "population" else 1)
assert not os.path.exists(f"/dev/shm/vllm_offload_{iid}.mmap")
+7 -8
View File
@@ -239,7 +239,7 @@ def test_hisparse_shares_host_pool_only_for_local_tp(monkeypatch):
@pytest.mark.skip_global_cleanup
def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch):
page = mmap.PAGESIZE
tp_group = MagicMock()
tp_group = MagicMock(rank_in_group=1, world_size=2)
monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", lambda: tp_group)
class FakeSharedOffloadRegion:
@@ -253,6 +253,7 @@ def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch):
self.pinned_addresses = []
self.view_sizes = []
self.offset = 0
kwargs["barrier"]()
def create_next_canonical_view(self, size):
self.view_sizes.append(size)
@@ -280,16 +281,13 @@ def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch):
pinned: list[torch.Tensor] = []
def pin_tensor(tensor):
tp_group.barrier.assert_called_once_with()
pinned.append(tensor)
monkeypatch.setattr(hisparse_runtime_module, "pin_tensor", pin_tensor)
config = SimpleNamespace(
instance_id="instance",
parallel_config=_hisparse_parallel_config(
tensor_parallel_size=1,
world_size=1,
distributed_executor_backend="uni",
),
parallel_config=_hisparse_parallel_config(),
)
pools, private_pools, region = hisparse_runtime_module.allocate_hisparse_host_pools(
@@ -310,7 +308,7 @@ def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch):
"cpu_page_size": 64,
"barrier": tp_group.barrier,
"creator_memory_check": hisparse_runtime_module.check_hisparse_host_memory,
"populate_only_on_creator": True,
"populate_shard": (1, 2),
}
assert region.view_sizes == [24, 40]
assert [pool.shape for pool in pools] == [(24,), (40,)]
@@ -328,7 +326,8 @@ def test_shared_host_pool_tracks_successful_registrations(
):
"""Only successful registrations may be unregistered on allocation failure."""
page = mmap.PAGESIZE
monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", MagicMock())
tp_group = MagicMock(rank_in_group=0, world_size=2)
monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", lambda: tp_group)
backing = torch.empty(3 * page, dtype=torch.uint8)
region = MagicMock(base_tensor=backing, pinned_addresses=[], is_pinned=False)
monkeypatch.setattr(
+3 -2
View File
@@ -367,6 +367,7 @@ def allocate_hisparse_host_pools(
None,
)
tp_group = get_tp_group()
region = SharedOffloadRegion(
engine_id=(
f"hisparse_{vllm_config.instance_id}_"
@@ -376,9 +377,9 @@ def allocate_hisparse_host_pools(
rank=0,
kv_bytes_per_chunk=num_blocks * host_block_stride,
cpu_page_size=sum(tensor_sizes),
barrier=get_tp_group().barrier,
barrier=tp_group.barrier,
creator_memory_check=check_hisparse_host_memory,
populate_only_on_creator=True,
populate_shard=(tp_group.rank_in_group, tp_group.world_size),
)
try:
for start, end in _hisparse_registration_ranges(
+35 -52
View File
@@ -81,10 +81,9 @@ class SharedOffloadRegion:
the kernel reclaims the memory when the last worker exits, no matter
how it exits; mappings taken before the unlink stay valid.
Creator-only population pre-faults the entire region before the barrier
and requires that barrier to keep joiners from using unpopulated pages.
``populate_shard=(index, count)`` pre-faults only this worker's contiguous
share, for layouts where every worker maps the same bytes.
share, for layouts where every worker maps the same bytes. The barrier
also waits for all shards to finish pre-faulting before returning.
"""
BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE
@@ -99,11 +98,8 @@ class SharedOffloadRegion:
barrier: Callable[[], None] | None = None,
*,
creator_memory_check: Callable[[int], None] | None = None,
populate_only_on_creator: bool = False,
populate_shard: tuple[int, int] | None = None,
) -> None:
if populate_only_on_creator and barrier is None:
raise ValueError("Creator-only population requires a barrier.")
self.page_size = mmap.PAGESIZE
assert kv_bytes_per_chunk % self.page_size == 0
@@ -119,6 +115,7 @@ class SharedOffloadRegion:
self._worker_offset = rank * cpu_page_size
# exclusive upper bound for this worker's area within each row
self._worker_area_end = (rank + 1) * cpu_page_size
barriers_remaining = 2 if populate_shard is not None else 1
try:
try:
self.fd: int | None = os.open(
@@ -155,75 +152,61 @@ class SharedOffloadRegion:
flags=mmap.MAP_SHARED,
prot=mmap.PROT_READ | mmap.PROT_WRITE,
)
if populate_only_on_creator and self._creator:
if barrier is not None:
barriers_remaining -= 1
barrier()
if self._creator:
os.unlink(self.mmap_path)
self._creator = False
logger.info("Unlinked mmap file %s", self.mmap_path)
if populate_shard is not None:
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
populate_write_fn(self.mmap_obj, 0, self.total_size_bytes)
index, count = populate_shard
share = round_up(cdiv(self.total_size_bytes, count), self.page_size)
start = min(index * share, self.total_size_bytes)
if start < self.total_size_bytes:
length = min(share, self.total_size_bytes - start)
populate_write_fn(self.mmap_obj, start, length)
if barrier is not None:
barriers_remaining -= 1
barrier()
except Exception:
if self._creator:
with contextlib.suppress(FileNotFoundError):
os.unlink(self.mmap_path)
self._creator = False
if hasattr(self, "mmap_obj") and self.mmap_obj is not None:
self.mmap_obj.close()
# The prefault traceback may retain an exported buffer.
with contextlib.suppress(BufferError):
self.mmap_obj.close()
self.mmap_obj = None
if hasattr(self, "fd") and self.fd is not None:
os.close(self.fd)
self.fd = None
# Peers block inside the barrier until the collective times out if
# we die before reaching it. Arrive anyway so every worker calls
# barrier() exactly once and they fail on their own errors instead
# of hanging; a failure here must not replace ours.
# Join remaining barriers so peers do not wait until the collective
# times out. A barrier failure must not replace the setup error.
if barrier is not None:
try:
barrier()
except Exception:
logger.warning(
"Failed to release peers waiting at the mmap barrier",
exc_info=True,
)
for _ in range(barriers_remaining):
try:
barrier()
except Exception:
logger.warning(
"Failed to release peers waiting at the mmap barrier",
exc_info=True,
)
raise
if barrier is not None:
# Every worker has mapped the file once the barrier releases, so
# its name is no longer needed and dropping it here means no exit
# path — including SIGKILL — can leak the file.
try:
barrier()
except Exception:
if self._creator:
with contextlib.suppress(FileNotFoundError):
os.unlink(self.mmap_path)
self._creator = False
self.mmap_obj.close()
os.close(self.fd)
self.mmap_obj = None
self.fd = None
raise
if self._creator:
os.unlink(self.mmap_path)
self._creator = False
logger.info("Unlinked mmap file %s", self.mmap_path)
self._base = torch.frombuffer(memoryview(self.mmap_obj), dtype=torch.int8)
self._views: list[torch.Tensor] = []
self._canonical_offset = 0
self.is_pinned: bool = False
self.pinned_addresses: list[int] = []
if populate_only_on_creator:
if populate_shard is not None:
return
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
if populate_shard is not None:
index, count = populate_shard
share = round_up(cdiv(self.total_size_bytes, count), self.page_size)
start = min(index * share, self.total_size_bytes)
if start < self.total_size_bytes:
length = min(share, self.total_size_bytes - start)
populate_write_fn(self.mmap_obj, start, length)
elif rank is not None:
if rank is not None:
# Populate only this worker's pages (one slot per chunk row).
worker_offset = rank * cpu_page_size
_t0 = time.perf_counter()