[HiSparse] Prefault the shared host pool in parallel (#60869)
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -217,7 +217,7 @@ def _mp_barrier_construct_and_hold(
|
||||
kv_bytes_per_chunk=PAGE_SIZE if replicated else num_workers * PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
barrier=lambda: barrier.wait(30),
|
||||
populate_only_on_creator=replicated,
|
||||
populate_shard=(rank, num_workers) if replicated else None,
|
||||
)
|
||||
# The constructor's barrier precedes the creator's unlink.
|
||||
barrier.wait(30)
|
||||
@@ -538,13 +538,23 @@ def test_populate_shard_prefaults_only_its_share(iid, monkeypatch, shard, expect
|
||||
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
||||
|
||||
madvise_calls: list[tuple[int, int]] = []
|
||||
monkeypatch.setattr(
|
||||
sor,
|
||||
"_madvise_populate_write",
|
||||
lambda mm, off, ln: madvise_calls.append((off, ln)),
|
||||
)
|
||||
with _region(iid, num_chunks=3, num_workers=2, populate_shard=shard):
|
||||
|
||||
def populate(mm, off, ln):
|
||||
assert not os.path.exists(f"/dev/shm/vllm_offload_{iid}.mmap")
|
||||
madvise_calls.append((off, ln))
|
||||
|
||||
monkeypatch.setattr(sor, "_madvise_populate_write", populate)
|
||||
|
||||
def check_population():
|
||||
if barrier.call_count == 2:
|
||||
assert madvise_calls == [(0, PAGE_SIZE), *expected]
|
||||
|
||||
barrier = MagicMock(side_effect=check_population)
|
||||
with _region(
|
||||
iid, num_chunks=3, num_workers=2, populate_shard=shard, barrier=barrier
|
||||
):
|
||||
assert madvise_calls == [(0, PAGE_SIZE), *expected]
|
||||
assert barrier.call_count == 2
|
||||
|
||||
|
||||
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
|
||||
@@ -1098,12 +1108,17 @@ def test_backing_file_unlinked_after_barrier(iid):
|
||||
_cleanup_file(path)
|
||||
|
||||
|
||||
def test_barrier_failure_unlinks_creator_and_raises(iid):
|
||||
@pytest.mark.parametrize("populate_shard", [None, (0, 1)])
|
||||
def test_barrier_failure_unlinks_creator_and_raises(iid, populate_shard):
|
||||
"""A failed rendezvous must remove the creator's file and re-raise, not
|
||||
leave a stub that wedges the next start in _wait_for_file_size."""
|
||||
path = f"/dev/shm/vllm_offload_{iid}.mmap"
|
||||
with pytest.raises(RuntimeError, match="peer died"):
|
||||
_make_region(iid, barrier=MagicMock(side_effect=RuntimeError("peer died")))
|
||||
_make_region(
|
||||
iid,
|
||||
barrier=MagicMock(side_effect=RuntimeError("peer died")),
|
||||
populate_shard=populate_shard,
|
||||
)
|
||||
assert not os.path.exists(path)
|
||||
|
||||
|
||||
@@ -1132,9 +1147,7 @@ def test_mp_barrier_unlinks_file_and_survives_sigkill(iid, replicated):
|
||||
assert len({r["inode"] for r in results}) == 1, "workers split onto two files"
|
||||
assert not any(r["path_exists"] for r in results)
|
||||
assert all(r["contents_match"] for r in results)
|
||||
assert sum(r["populate_calls"] for r in results) == (
|
||||
1 if replicated else num_workers
|
||||
)
|
||||
assert sum(r["populate_calls"] for r in results) == num_workers
|
||||
assert not os.path.exists(path)
|
||||
finally:
|
||||
for p in procs:
|
||||
@@ -1146,23 +1159,40 @@ def test_mp_barrier_unlinks_file_and_survives_sigkill(iid, replicated):
|
||||
assert restarted._creator is True, "restart must be able to create anew"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure_phase", ["shm", "memory", "population"])
|
||||
def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch, failure_phase):
|
||||
@pytest.mark.parametrize(
|
||||
"failure_phase,populate_shard",
|
||||
[("shm", None), ("memory", None)]
|
||||
+ [
|
||||
(phase, (0, 1))
|
||||
for phase in ("shm", "memory", "probe", "population", "population_view")
|
||||
],
|
||||
)
|
||||
def test_setup_failure_before_barrier_releases_peers(
|
||||
iid, monkeypatch, failure_phase, populate_shard
|
||||
):
|
||||
"""A worker that dies before the rendezvous must still arrive at the
|
||||
barrier, or its peers block in the collective until it times out."""
|
||||
failure = MagicMock(side_effect=ValueError("Insufficient space"))
|
||||
failure = MagicMock(side_effect=ValueError("setup failed"))
|
||||
memory_check = failure if failure_phase == "memory" else None
|
||||
if failure_phase == "shm":
|
||||
monkeypatch.setattr(region_module, "check_shm_free_space", failure)
|
||||
elif failure_phase == "population":
|
||||
monkeypatch.setattr(region_module, "_get_populate_write_fn", failure)
|
||||
elif failure_phase in ("probe", "population", "population_view"):
|
||||
|
||||
def populate(mm, offset, length):
|
||||
failure(memoryview(mm) if failure_phase == "population_view" else None)
|
||||
|
||||
monkeypatch.setattr(
|
||||
region_module,
|
||||
"_get_populate_write_fn",
|
||||
failure if failure_phase == "probe" else lambda mm: populate,
|
||||
)
|
||||
path = f"/dev/shm/vllm_offload_{iid}.mmap"
|
||||
seen_at_barrier = []
|
||||
barrier = MagicMock(
|
||||
side_effect=lambda: seen_at_barrier.append(os.path.exists(path))
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Insufficient space"):
|
||||
with pytest.raises(ValueError, match="setup failed"):
|
||||
SharedOffloadRegion(
|
||||
engine_id=iid,
|
||||
num_chunks=1,
|
||||
@@ -1171,11 +1201,15 @@ def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch, failure_p
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
barrier=barrier,
|
||||
creator_memory_check=memory_check,
|
||||
populate_only_on_creator=True,
|
||||
populate_shard=populate_shard,
|
||||
)
|
||||
|
||||
barrier.assert_called_once_with()
|
||||
assert seen_at_barrier == [False]
|
||||
assert barrier.call_count == (2 if populate_shard is not None else 1)
|
||||
assert seen_at_barrier == (
|
||||
[False] * barrier.call_count
|
||||
if failure_phase in ("shm", "memory")
|
||||
else [True, False]
|
||||
)
|
||||
|
||||
|
||||
def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch):
|
||||
@@ -1191,18 +1225,29 @@ def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch):
|
||||
assert seen_at_barrier == [False], "file must be gone before peers release"
|
||||
|
||||
|
||||
def test_barrier_release_failure_keeps_original_error(iid, monkeypatch):
|
||||
@pytest.mark.parametrize("failure_phase", ["shm", "population"])
|
||||
def test_barrier_release_failure_keeps_original_error(iid, monkeypatch, failure_phase):
|
||||
"""When releasing the peers fails too, the setup error that explains the
|
||||
failure must be the one that propagates."""
|
||||
import vllm.v1.kv_offload.cpu.shared_offload_region as region
|
||||
|
||||
monkeypatch.setattr(
|
||||
region,
|
||||
"check_shm_free_space",
|
||||
MagicMock(side_effect=RuntimeError("Insufficient space")),
|
||||
failure = MagicMock(side_effect=RuntimeError("setup failed"))
|
||||
if failure_phase == "shm":
|
||||
monkeypatch.setattr(region, "check_shm_free_space", failure)
|
||||
else:
|
||||
monkeypatch.setattr(region, "_get_populate_write_fn", lambda mm: failure)
|
||||
|
||||
barrier = MagicMock(
|
||||
side_effect=[None, TimeoutError("barrier")]
|
||||
if failure_phase == "population"
|
||||
else TimeoutError("barrier")
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="setup failed"):
|
||||
_make_region(
|
||||
iid,
|
||||
barrier=barrier,
|
||||
populate_shard=(0, 1) if failure_phase == "population" else None,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Insufficient space"):
|
||||
_make_region(iid, barrier=MagicMock(side_effect=TimeoutError("barrier")))
|
||||
|
||||
assert barrier.call_count == (2 if failure_phase == "population" else 1)
|
||||
assert not os.path.exists(f"/dev/shm/vllm_offload_{iid}.mmap")
|
||||
|
||||
@@ -239,7 +239,7 @@ def test_hisparse_shares_host_pool_only_for_local_tp(monkeypatch):
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch):
|
||||
page = mmap.PAGESIZE
|
||||
tp_group = MagicMock()
|
||||
tp_group = MagicMock(rank_in_group=1, world_size=2)
|
||||
monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", lambda: tp_group)
|
||||
|
||||
class FakeSharedOffloadRegion:
|
||||
@@ -253,6 +253,7 @@ def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch):
|
||||
self.pinned_addresses = []
|
||||
self.view_sizes = []
|
||||
self.offset = 0
|
||||
kwargs["barrier"]()
|
||||
|
||||
def create_next_canonical_view(self, size):
|
||||
self.view_sizes.append(size)
|
||||
@@ -280,16 +281,13 @@ def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch):
|
||||
pinned: list[torch.Tensor] = []
|
||||
|
||||
def pin_tensor(tensor):
|
||||
tp_group.barrier.assert_called_once_with()
|
||||
pinned.append(tensor)
|
||||
|
||||
monkeypatch.setattr(hisparse_runtime_module, "pin_tensor", pin_tensor)
|
||||
config = SimpleNamespace(
|
||||
instance_id="instance",
|
||||
parallel_config=_hisparse_parallel_config(
|
||||
tensor_parallel_size=1,
|
||||
world_size=1,
|
||||
distributed_executor_backend="uni",
|
||||
),
|
||||
parallel_config=_hisparse_parallel_config(),
|
||||
)
|
||||
|
||||
pools, private_pools, region = hisparse_runtime_module.allocate_hisparse_host_pools(
|
||||
@@ -310,7 +308,7 @@ def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch):
|
||||
"cpu_page_size": 64,
|
||||
"barrier": tp_group.barrier,
|
||||
"creator_memory_check": hisparse_runtime_module.check_hisparse_host_memory,
|
||||
"populate_only_on_creator": True,
|
||||
"populate_shard": (1, 2),
|
||||
}
|
||||
assert region.view_sizes == [24, 40]
|
||||
assert [pool.shape for pool in pools] == [(24,), (40,)]
|
||||
@@ -328,7 +326,8 @@ def test_shared_host_pool_tracks_successful_registrations(
|
||||
):
|
||||
"""Only successful registrations may be unregistered on allocation failure."""
|
||||
page = mmap.PAGESIZE
|
||||
monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", MagicMock())
|
||||
tp_group = MagicMock(rank_in_group=0, world_size=2)
|
||||
monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", lambda: tp_group)
|
||||
backing = torch.empty(3 * page, dtype=torch.uint8)
|
||||
region = MagicMock(base_tensor=backing, pinned_addresses=[], is_pinned=False)
|
||||
monkeypatch.setattr(
|
||||
|
||||
@@ -367,6 +367,7 @@ def allocate_hisparse_host_pools(
|
||||
None,
|
||||
)
|
||||
|
||||
tp_group = get_tp_group()
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=(
|
||||
f"hisparse_{vllm_config.instance_id}_"
|
||||
@@ -376,9 +377,9 @@ def allocate_hisparse_host_pools(
|
||||
rank=0,
|
||||
kv_bytes_per_chunk=num_blocks * host_block_stride,
|
||||
cpu_page_size=sum(tensor_sizes),
|
||||
barrier=get_tp_group().barrier,
|
||||
barrier=tp_group.barrier,
|
||||
creator_memory_check=check_hisparse_host_memory,
|
||||
populate_only_on_creator=True,
|
||||
populate_shard=(tp_group.rank_in_group, tp_group.world_size),
|
||||
)
|
||||
try:
|
||||
for start, end in _hisparse_registration_ranges(
|
||||
|
||||
@@ -81,10 +81,9 @@ class SharedOffloadRegion:
|
||||
the kernel reclaims the memory when the last worker exits, no matter
|
||||
how it exits; mappings taken before the unlink stay valid.
|
||||
|
||||
Creator-only population pre-faults the entire region before the barrier
|
||||
and requires that barrier to keep joiners from using unpopulated pages.
|
||||
``populate_shard=(index, count)`` pre-faults only this worker's contiguous
|
||||
share, for layouts where every worker maps the same bytes.
|
||||
share, for layouts where every worker maps the same bytes. The barrier
|
||||
also waits for all shards to finish pre-faulting before returning.
|
||||
"""
|
||||
|
||||
BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE
|
||||
@@ -99,11 +98,8 @@ class SharedOffloadRegion:
|
||||
barrier: Callable[[], None] | None = None,
|
||||
*,
|
||||
creator_memory_check: Callable[[int], None] | None = None,
|
||||
populate_only_on_creator: bool = False,
|
||||
populate_shard: tuple[int, int] | None = None,
|
||||
) -> None:
|
||||
if populate_only_on_creator and barrier is None:
|
||||
raise ValueError("Creator-only population requires a barrier.")
|
||||
self.page_size = mmap.PAGESIZE
|
||||
assert kv_bytes_per_chunk % self.page_size == 0
|
||||
|
||||
@@ -119,6 +115,7 @@ class SharedOffloadRegion:
|
||||
self._worker_offset = rank * cpu_page_size
|
||||
# exclusive upper bound for this worker's area within each row
|
||||
self._worker_area_end = (rank + 1) * cpu_page_size
|
||||
barriers_remaining = 2 if populate_shard is not None else 1
|
||||
try:
|
||||
try:
|
||||
self.fd: int | None = os.open(
|
||||
@@ -155,75 +152,61 @@ class SharedOffloadRegion:
|
||||
flags=mmap.MAP_SHARED,
|
||||
prot=mmap.PROT_READ | mmap.PROT_WRITE,
|
||||
)
|
||||
|
||||
if populate_only_on_creator and self._creator:
|
||||
if barrier is not None:
|
||||
barriers_remaining -= 1
|
||||
barrier()
|
||||
if self._creator:
|
||||
os.unlink(self.mmap_path)
|
||||
self._creator = False
|
||||
logger.info("Unlinked mmap file %s", self.mmap_path)
|
||||
if populate_shard is not None:
|
||||
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
|
||||
populate_write_fn(self.mmap_obj, 0, self.total_size_bytes)
|
||||
index, count = populate_shard
|
||||
share = round_up(cdiv(self.total_size_bytes, count), self.page_size)
|
||||
start = min(index * share, self.total_size_bytes)
|
||||
if start < self.total_size_bytes:
|
||||
length = min(share, self.total_size_bytes - start)
|
||||
populate_write_fn(self.mmap_obj, start, length)
|
||||
if barrier is not None:
|
||||
barriers_remaining -= 1
|
||||
barrier()
|
||||
except Exception:
|
||||
if self._creator:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(self.mmap_path)
|
||||
self._creator = False
|
||||
if hasattr(self, "mmap_obj") and self.mmap_obj is not None:
|
||||
self.mmap_obj.close()
|
||||
# The prefault traceback may retain an exported buffer.
|
||||
with contextlib.suppress(BufferError):
|
||||
self.mmap_obj.close()
|
||||
self.mmap_obj = None
|
||||
if hasattr(self, "fd") and self.fd is not None:
|
||||
os.close(self.fd)
|
||||
self.fd = None
|
||||
# Peers block inside the barrier until the collective times out if
|
||||
# we die before reaching it. Arrive anyway so every worker calls
|
||||
# barrier() exactly once and they fail on their own errors instead
|
||||
# of hanging; a failure here must not replace ours.
|
||||
# Join remaining barriers so peers do not wait until the collective
|
||||
# times out. A barrier failure must not replace the setup error.
|
||||
if barrier is not None:
|
||||
try:
|
||||
barrier()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to release peers waiting at the mmap barrier",
|
||||
exc_info=True,
|
||||
)
|
||||
for _ in range(barriers_remaining):
|
||||
try:
|
||||
barrier()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to release peers waiting at the mmap barrier",
|
||||
exc_info=True,
|
||||
)
|
||||
raise
|
||||
|
||||
if barrier is not None:
|
||||
# Every worker has mapped the file once the barrier releases, so
|
||||
# its name is no longer needed and dropping it here means no exit
|
||||
# path — including SIGKILL — can leak the file.
|
||||
try:
|
||||
barrier()
|
||||
except Exception:
|
||||
if self._creator:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(self.mmap_path)
|
||||
self._creator = False
|
||||
self.mmap_obj.close()
|
||||
os.close(self.fd)
|
||||
self.mmap_obj = None
|
||||
self.fd = None
|
||||
raise
|
||||
if self._creator:
|
||||
os.unlink(self.mmap_path)
|
||||
self._creator = False
|
||||
logger.info("Unlinked mmap file %s", self.mmap_path)
|
||||
|
||||
self._base = torch.frombuffer(memoryview(self.mmap_obj), dtype=torch.int8)
|
||||
self._views: list[torch.Tensor] = []
|
||||
self._canonical_offset = 0
|
||||
self.is_pinned: bool = False
|
||||
self.pinned_addresses: list[int] = []
|
||||
|
||||
if populate_only_on_creator:
|
||||
if populate_shard is not None:
|
||||
return
|
||||
|
||||
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
|
||||
|
||||
if populate_shard is not None:
|
||||
index, count = populate_shard
|
||||
share = round_up(cdiv(self.total_size_bytes, count), self.page_size)
|
||||
start = min(index * share, self.total_size_bytes)
|
||||
if start < self.total_size_bytes:
|
||||
length = min(share, self.total_size_bytes - start)
|
||||
populate_write_fn(self.mmap_obj, start, length)
|
||||
elif rank is not None:
|
||||
if rank is not None:
|
||||
# Populate only this worker's pages (one slot per chunk row).
|
||||
worker_offset = rank * cpu_page_size
|
||||
_t0 = time.perf_counter()
|
||||
|
||||
Reference in New Issue
Block a user