[HiSparse] Prefault the shared host pool in parallel (#60869)
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -217,7 +217,7 @@ def _mp_barrier_construct_and_hold(
|
||||
kv_bytes_per_chunk=PAGE_SIZE if replicated else num_workers * PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
barrier=lambda: barrier.wait(30),
|
||||
populate_only_on_creator=replicated,
|
||||
populate_shard=(rank, num_workers) if replicated else None,
|
||||
)
|
||||
# The constructor's barrier precedes the creator's unlink.
|
||||
barrier.wait(30)
|
||||
@@ -538,13 +538,23 @@ def test_populate_shard_prefaults_only_its_share(iid, monkeypatch, shard, expect
|
||||
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
||||
|
||||
madvise_calls: list[tuple[int, int]] = []
|
||||
monkeypatch.setattr(
|
||||
sor,
|
||||
"_madvise_populate_write",
|
||||
lambda mm, off, ln: madvise_calls.append((off, ln)),
|
||||
)
|
||||
with _region(iid, num_chunks=3, num_workers=2, populate_shard=shard):
|
||||
|
||||
def populate(mm, off, ln):
|
||||
assert not os.path.exists(f"/dev/shm/vllm_offload_{iid}.mmap")
|
||||
madvise_calls.append((off, ln))
|
||||
|
||||
monkeypatch.setattr(sor, "_madvise_populate_write", populate)
|
||||
|
||||
def check_population():
|
||||
if barrier.call_count == 2:
|
||||
assert madvise_calls == [(0, PAGE_SIZE), *expected]
|
||||
|
||||
barrier = MagicMock(side_effect=check_population)
|
||||
with _region(
|
||||
iid, num_chunks=3, num_workers=2, populate_shard=shard, barrier=barrier
|
||||
):
|
||||
assert madvise_calls == [(0, PAGE_SIZE), *expected]
|
||||
assert barrier.call_count == 2
|
||||
|
||||
|
||||
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
|
||||
@@ -1098,12 +1108,17 @@ def test_backing_file_unlinked_after_barrier(iid):
|
||||
_cleanup_file(path)
|
||||
|
||||
|
||||
def test_barrier_failure_unlinks_creator_and_raises(iid):
|
||||
@pytest.mark.parametrize("populate_shard", [None, (0, 1)])
|
||||
def test_barrier_failure_unlinks_creator_and_raises(iid, populate_shard):
|
||||
"""A failed rendezvous must remove the creator's file and re-raise, not
|
||||
leave a stub that wedges the next start in _wait_for_file_size."""
|
||||
path = f"/dev/shm/vllm_offload_{iid}.mmap"
|
||||
with pytest.raises(RuntimeError, match="peer died"):
|
||||
_make_region(iid, barrier=MagicMock(side_effect=RuntimeError("peer died")))
|
||||
_make_region(
|
||||
iid,
|
||||
barrier=MagicMock(side_effect=RuntimeError("peer died")),
|
||||
populate_shard=populate_shard,
|
||||
)
|
||||
assert not os.path.exists(path)
|
||||
|
||||
|
||||
@@ -1132,9 +1147,7 @@ def test_mp_barrier_unlinks_file_and_survives_sigkill(iid, replicated):
|
||||
assert len({r["inode"] for r in results}) == 1, "workers split onto two files"
|
||||
assert not any(r["path_exists"] for r in results)
|
||||
assert all(r["contents_match"] for r in results)
|
||||
assert sum(r["populate_calls"] for r in results) == (
|
||||
1 if replicated else num_workers
|
||||
)
|
||||
assert sum(r["populate_calls"] for r in results) == num_workers
|
||||
assert not os.path.exists(path)
|
||||
finally:
|
||||
for p in procs:
|
||||
@@ -1146,23 +1159,40 @@ def test_mp_barrier_unlinks_file_and_survives_sigkill(iid, replicated):
|
||||
assert restarted._creator is True, "restart must be able to create anew"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure_phase", ["shm", "memory", "population"])
|
||||
def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch, failure_phase):
|
||||
@pytest.mark.parametrize(
|
||||
"failure_phase,populate_shard",
|
||||
[("shm", None), ("memory", None)]
|
||||
+ [
|
||||
(phase, (0, 1))
|
||||
for phase in ("shm", "memory", "probe", "population", "population_view")
|
||||
],
|
||||
)
|
||||
def test_setup_failure_before_barrier_releases_peers(
|
||||
iid, monkeypatch, failure_phase, populate_shard
|
||||
):
|
||||
"""A worker that dies before the rendezvous must still arrive at the
|
||||
barrier, or its peers block in the collective until it times out."""
|
||||
failure = MagicMock(side_effect=ValueError("Insufficient space"))
|
||||
failure = MagicMock(side_effect=ValueError("setup failed"))
|
||||
memory_check = failure if failure_phase == "memory" else None
|
||||
if failure_phase == "shm":
|
||||
monkeypatch.setattr(region_module, "check_shm_free_space", failure)
|
||||
elif failure_phase == "population":
|
||||
monkeypatch.setattr(region_module, "_get_populate_write_fn", failure)
|
||||
elif failure_phase in ("probe", "population", "population_view"):
|
||||
|
||||
def populate(mm, offset, length):
|
||||
failure(memoryview(mm) if failure_phase == "population_view" else None)
|
||||
|
||||
monkeypatch.setattr(
|
||||
region_module,
|
||||
"_get_populate_write_fn",
|
||||
failure if failure_phase == "probe" else lambda mm: populate,
|
||||
)
|
||||
path = f"/dev/shm/vllm_offload_{iid}.mmap"
|
||||
seen_at_barrier = []
|
||||
barrier = MagicMock(
|
||||
side_effect=lambda: seen_at_barrier.append(os.path.exists(path))
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Insufficient space"):
|
||||
with pytest.raises(ValueError, match="setup failed"):
|
||||
SharedOffloadRegion(
|
||||
engine_id=iid,
|
||||
num_chunks=1,
|
||||
@@ -1171,11 +1201,15 @@ def test_setup_failure_before_barrier_releases_peers(iid, monkeypatch, failure_p
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
barrier=barrier,
|
||||
creator_memory_check=memory_check,
|
||||
populate_only_on_creator=True,
|
||||
populate_shard=populate_shard,
|
||||
)
|
||||
|
||||
barrier.assert_called_once_with()
|
||||
assert seen_at_barrier == [False]
|
||||
assert barrier.call_count == (2 if populate_shard is not None else 1)
|
||||
assert seen_at_barrier == (
|
||||
[False] * barrier.call_count
|
||||
if failure_phase in ("shm", "memory")
|
||||
else [True, False]
|
||||
)
|
||||
|
||||
|
||||
def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch):
|
||||
@@ -1191,18 +1225,29 @@ def test_mmap_failure_unlinks_creator_before_releasing_peers(iid, monkeypatch):
|
||||
assert seen_at_barrier == [False], "file must be gone before peers release"
|
||||
|
||||
|
||||
def test_barrier_release_failure_keeps_original_error(iid, monkeypatch):
|
||||
@pytest.mark.parametrize("failure_phase", ["shm", "population"])
|
||||
def test_barrier_release_failure_keeps_original_error(iid, monkeypatch, failure_phase):
|
||||
"""When releasing the peers fails too, the setup error that explains the
|
||||
failure must be the one that propagates."""
|
||||
import vllm.v1.kv_offload.cpu.shared_offload_region as region
|
||||
|
||||
monkeypatch.setattr(
|
||||
region,
|
||||
"check_shm_free_space",
|
||||
MagicMock(side_effect=RuntimeError("Insufficient space")),
|
||||
failure = MagicMock(side_effect=RuntimeError("setup failed"))
|
||||
if failure_phase == "shm":
|
||||
monkeypatch.setattr(region, "check_shm_free_space", failure)
|
||||
else:
|
||||
monkeypatch.setattr(region, "_get_populate_write_fn", lambda mm: failure)
|
||||
|
||||
barrier = MagicMock(
|
||||
side_effect=[None, TimeoutError("barrier")]
|
||||
if failure_phase == "population"
|
||||
else TimeoutError("barrier")
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="setup failed"):
|
||||
_make_region(
|
||||
iid,
|
||||
barrier=barrier,
|
||||
populate_shard=(0, 1) if failure_phase == "population" else None,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Insufficient space"):
|
||||
_make_region(iid, barrier=MagicMock(side_effect=TimeoutError("barrier")))
|
||||
|
||||
assert barrier.call_count == (2 if failure_phase == "population" else 1)
|
||||
assert not os.path.exists(f"/dev/shm/vllm_offload_{iid}.mmap")
|
||||
|
||||
@@ -239,7 +239,7 @@ def test_hisparse_shares_host_pool_only_for_local_tp(monkeypatch):
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch):
|
||||
page = mmap.PAGESIZE
|
||||
tp_group = MagicMock()
|
||||
tp_group = MagicMock(rank_in_group=1, world_size=2)
|
||||
monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", lambda: tp_group)
|
||||
|
||||
class FakeSharedOffloadRegion:
|
||||
@@ -253,6 +253,7 @@ def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch):
|
||||
self.pinned_addresses = []
|
||||
self.view_sizes = []
|
||||
self.offset = 0
|
||||
kwargs["barrier"]()
|
||||
|
||||
def create_next_canonical_view(self, size):
|
||||
self.view_sizes.append(size)
|
||||
@@ -280,16 +281,13 @@ def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch):
|
||||
pinned: list[torch.Tensor] = []
|
||||
|
||||
def pin_tensor(tensor):
|
||||
tp_group.barrier.assert_called_once_with()
|
||||
pinned.append(tensor)
|
||||
|
||||
monkeypatch.setattr(hisparse_runtime_module, "pin_tensor", pin_tensor)
|
||||
config = SimpleNamespace(
|
||||
instance_id="instance",
|
||||
parallel_config=_hisparse_parallel_config(
|
||||
tensor_parallel_size=1,
|
||||
world_size=1,
|
||||
distributed_executor_backend="uni",
|
||||
),
|
||||
parallel_config=_hisparse_parallel_config(),
|
||||
)
|
||||
|
||||
pools, private_pools, region = hisparse_runtime_module.allocate_hisparse_host_pools(
|
||||
@@ -310,7 +308,7 @@ def test_hisparse_shared_host_pool_uses_one_replicated_mmap(monkeypatch):
|
||||
"cpu_page_size": 64,
|
||||
"barrier": tp_group.barrier,
|
||||
"creator_memory_check": hisparse_runtime_module.check_hisparse_host_memory,
|
||||
"populate_only_on_creator": True,
|
||||
"populate_shard": (1, 2),
|
||||
}
|
||||
assert region.view_sizes == [24, 40]
|
||||
assert [pool.shape for pool in pools] == [(24,), (40,)]
|
||||
@@ -328,7 +326,8 @@ def test_shared_host_pool_tracks_successful_registrations(
|
||||
):
|
||||
"""Only successful registrations may be unregistered on allocation failure."""
|
||||
page = mmap.PAGESIZE
|
||||
monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", MagicMock())
|
||||
tp_group = MagicMock(rank_in_group=0, world_size=2)
|
||||
monkeypatch.setattr(hisparse_runtime_module, "get_tp_group", lambda: tp_group)
|
||||
backing = torch.empty(3 * page, dtype=torch.uint8)
|
||||
region = MagicMock(base_tensor=backing, pinned_addresses=[], is_pinned=False)
|
||||
monkeypatch.setattr(
|
||||
|
||||
Reference in New Issue
Block a user